From c14f675a335b980f95e5f72c7a1951637d5a3d41 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 3 Jul 2026 16:12:45 -0400 Subject: [PATCH 1/2] feat: Add pipecat integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds observer-based Pipecat instrumentation that injects a BraintrustPipecatObserver into PipelineWorker construction via setup_pipecat() and auto_instrument(). The observer turns real Pipecat pipeline/provider events into Braintrust task and llm spans, with VCR-backed OpenAI/Pipecat coverage for both latest and 1.3.0. Try it in an app: ```python import braintrust from pipecat.frames.frames import EndFrame, LLMContextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.services.openai.llm import OpenAILLMService from pipecat.workers.runner import WorkerRunner braintrust.init_logger(project="pipecat-demo") braintrust.auto_instrument() # installs the Pipecat PipelineWorker observer llm = OpenAILLMService( api_key="...", settings=OpenAILLMService.Settings( model="gpt-4o-mini", temperature=0.0, max_completion_tokens=64, ), ) worker = PipelineWorker( Pipeline([llm]), params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), ) context = LLMContext(messages=[{"role": "user", "content": "Say hello"}]) @worker.event_handler("on_pipeline_started") async def on_pipeline_started(_worker, _frame): await worker.queue_frames([LLMContextFrame(context), EndFrame()]) runner = WorkerRunner(handle_sigint=False) await runner.add_workers(worker) await runner.run() ``` To validate locally from this repo: ```bash cd py nox -s "test_pipecat(latest)" nox -s "test_pipecat(1.3.0)" OPENAI_API_KEY=sk-test-dummy-api-key-for-vcr-tests \ nox -s "test_pipecat(latest)" -- -k test_setup_pipecat_traces_real_pipeline_frames ``` Expected trace shape: ```text pipecat_pipeline task └── pipecat_llm_response llm ├── input: [{role: developer}, {role: user, ...}] ├── output: [{index: 0, finish_reason: stop, message: {...}}] ├── metadata: {provider: openai, model: gpt-4o-mini} └── metrics: {prompt_tokens, completion_tokens, tokens, time_to_first_token} ``` --- docs/pipecat-ai-integration-plan.md | 293 +++++++++ py/noxfile.py | 16 + py/pyproject.toml | 12 + py/src/braintrust/auto.py | 5 + py/src/braintrust/integrations/__init__.py | 2 + .../auto_test_scripts/test_auto_pipecat.py | 102 ++++ .../integrations/pipecat/__init__.py | 38 ++ ...st_auto_instrument_pipecat_subprocess.yaml | 2 + .../cassettes/1.3.0/test_auto_pipecat.yaml | 131 ++++ ...d_wrap_pipeline_worker_are_idempotent.yaml | 2 + ...p_pipecat_traces_real_pipeline_frames.yaml | 132 ++++ ...st_auto_instrument_pipecat_subprocess.yaml | 2 + .../cassettes/latest/test_auto_pipecat.yaml | 131 ++++ ...d_wrap_pipeline_worker_are_idempotent.yaml | 2 + ...p_pipecat_traces_real_pipeline_frames.yaml | 132 ++++ .../integrations/pipecat/integration.py | 15 + .../integrations/pipecat/patchers.py | 51 ++ .../integrations/pipecat/test_pipecat.py | 180 ++++++ .../integrations/pipecat/tracing.py | 564 ++++++++++++++++++ py/uv.lock | 105 +--- 20 files changed, 1829 insertions(+), 88 deletions(-) create mode 100644 docs/pipecat-ai-integration-plan.md create mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py create mode 100644 py/src/braintrust/integrations/pipecat/__init__.py create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_instrument_pipecat_subprocess.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_pipecat.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_pipecat_traces_real_pipeline_frames.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_instrument_pipecat_subprocess.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_pipecat.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_pipecat_traces_real_pipeline_frames.yaml create mode 100644 py/src/braintrust/integrations/pipecat/integration.py create mode 100644 py/src/braintrust/integrations/pipecat/patchers.py create mode 100644 py/src/braintrust/integrations/pipecat/test_pipecat.py create mode 100644 py/src/braintrust/integrations/pipecat/tracing.py diff --git a/docs/pipecat-ai-integration-plan.md b/docs/pipecat-ai-integration-plan.md new file mode 100644 index 00000000..d7648114 --- /dev/null +++ b/docs/pipecat-ai-integration-plan.md @@ -0,0 +1,293 @@ +# Pipecat AI Integration Research and Plan + +Research source: `https://github.com/pipecat-ai/pipecat`, cloned locally at `/tmp/pipecat`, commit `81f5be1e638188f9fbd3f386accc360e0db934d5` (main after `v1.4.0`). + +Braintrust instrumentation spec source: `https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md`, reviewed 2026-07-03. + +## Summary + +Pipecat is a real-time voice/multimodal agent framework. It is not a single model provider SDK; it orchestrates transports, STT, LLM, TTS, tool calls, metrics, workers, and pipelines through a frame-processing graph. + +The best Braintrust integration point is Pipecat's observer system, installed automatically by patching worker construction. This captures the framework-level semantics users care about without wrapping every provider-specific Pipecat service implementation. + +Recommended initial integration: + +- Package: `braintrust.integrations.pipecat` +- Import/distribution: `pipecat` / `pipecat-ai` +- Minimum supported Pipecat version: `1.3.0` +- Patch target: `pipecat.pipeline.worker.PipelineWorker.__init__` +- Behavior: inject a Braintrust `BaseObserver` into the worker's `observers` list; the observer creates Braintrust spans from Pipecat frames and metrics. + +## Relevant Pipecat APIs + +### Public orchestration APIs + +Current Pipecat applications are built around: + +- `pipecat.pipeline.pipeline.Pipeline` +- `pipecat.pipeline.worker.PipelineWorker` +- `pipecat.pipeline.worker.PipelineParams` +- `pipecat.workers.runner.WorkerRunner` +- `pipecat.observers.base_observer.BaseObserver` +- `pipecat.observers.base_observer.FramePushed` +- `pipecat.observers.base_observer.FrameProcessed` + +`PipelineWorker` accepts `observers: list[BaseObserver] | None` and forwards frame activity through a `WorkerObserver` proxy. This is the cleanest hook because it is public, non-invasive, and already intended for analytics/monitoring. + +### Frame events worth tracing + +Important frame classes in `src/pipecat/frames/frames.py`: + +- Pipeline lifecycle: `StartFrame`, `EndFrame`, `StopFrame`, `CancelFrame`, `ErrorFrame` +- Turn and voice activity: `UserStartedSpeakingFrame`, `UserStoppedSpeakingFrame`, `BotStartedSpeakingFrame`, `BotStoppedSpeakingFrame`, `InterruptionFrame` +- STT: `TranscriptionFrame`, `InterimTranscriptionFrame` +- LLM: `LLMContextFrame`, `LLMFullResponseStartFrame`, `LLMTextFrame`, `LLMFullResponseEndFrame` +- Tool calls: `FunctionCallsStartedFrame`, `FunctionCallInProgressFrame`, `FunctionCallResultFrame`, `FunctionCallCancelFrame` +- TTS/audio: `TTSStartedFrame`, `TTSTextFrame`, `TTSAudioRawFrame`, `TTSStoppedFrame` +- Metrics: `MetricsFrame` containing `TTFBMetricsData`, `TTFAMetricsData`, `ProcessingMetricsData`, `LLMUsageMetricsData`, `TTSUsageMetricsData`, `TextAggregationMetricsData`, `TurnMetricsData` + +### Existing Pipecat observability + +Pipecat already ships: + +- Observer API (`BaseObserver`) for non-intrusive frame monitoring. +- Logging observers (`LLMLogObserver`, `MetricsLogObserver`, `TranscriptionLogObserver`). +- Built-in OpenTelemetry tracing via `PipelineWorker(enable_tracing=True)` and `pipecat-ai[tracing]`. + +The Braintrust integration should use observers and native Braintrust logging as the primary implementation. Pipecat's OpenTelemetry support can be a future bridge, but it should not be required for the initial integration because it adds optional dependencies and mostly emits OTel attributes rather than Braintrust-shaped span `input`/`output`/`metadata`/`metrics`. If a future OTel bridge is added, it must use the spec's `braintrust.input_json`, `braintrust.output_json`, `braintrust.metadata`, `braintrust.metrics`, and `braintrust.span_attributes` attributes rather than ad-hoc OTel fields. + +Avoid excess normalization in integration code. Braintrust's span processing / serialization pipeline already normalizes objects before sending them to Braintrust, so the Pipecat integration should shape payloads into the required high-level schema but not defensively JSON serialize/deserialize, recursively convert every provider object, or stringify values just to make them serializable. + +## Minimum version recommendation + +Use `pipecat-ai>=1.3.0`. + +Reasoning: + +| Version | Finding | +| --- | --- | +| `0.0.108` | Last pre-1.x tag; supports Python `>=3.10`, has `PipelineTask`, observers, and tracing, but is pre-stable and has many APIs later deprecated/renamed. | +| `1.0.0` | First stable release; requires Python `>=3.11`; still centered on `PipelineTask` / `PipelineRunner`. | +| `1.2.0` | Still `PipelineTask` based; adds/changes resource APIs. | +| `1.3.0` | Introduces current `PipelineWorker` and `WorkerRunner`; `PipelineTask` / `PipelineRunner` become deprecated aliases. | +| `1.4.0` | Current stable tag in clone; keeps `PipelineWorker` observer API and adds more frame/metrics features. | + +Supporting `1.0.0` through `1.2.x` would require a parallel `PipelineTask.__init__` patcher for deprecated APIs. That adds complexity for APIs Pipecat itself moved away from. For a new Braintrust integration, start at the current worker architecture (`1.3.0+`). + +Implications for Braintrust test matrix: + +- Add `[tool.braintrust.matrix.pipecat-ai]` with `latest = "pipecat-ai==1.4.0"` and floor `"1.3.0" = "pipecat-ai==1.3.0"`. +- Add `pipecat-ai = "pipecat"` to `[tool.braintrust.vendor-packages]`. +- Skip the nox session on Python `<3.11` because Pipecat 1.x requires Python `>=3.11`. + +## Braintrust instrumentation spec alignment + +The Braintrust instrumentation guide constrains the Pipecat design in a few important ways: + +- Pipecat is an **agentic / realtime framework**, not a provider. The outer Pipecat run should be a `task` span. Individual model calls inside the run should be child `llm` spans, and tool executions should be child `tool` spans. +- Any `llm` span emitted by the Pipecat integration must use the completion API schema: + - `input`: ordered OpenAI-style message array, unless a provider-native normalizer is intentionally used. Pipecat's `LLMContext` already uses an OpenAI-like universal message format, so use `context.get_messages()` and materialize attachments. + - `output`: OpenAI Chat Completions-style choices array, e.g. `[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "..."}}]`. + - tool-call outputs: OpenAI-style `message.tool_calls`, with function `arguments` as a JSON-encoded string. + - `metadata`: must include `model` and `provider` when known, plus only allowed LLM request configuration fields, `tools`, `tool_choice`, and prompt provenance. +- Available tool definitions are **metadata**, not input messages. Convert Pipecat `ToolsSchema.standard_tools` / `FunctionSchema` / direct-function schemas to OpenAI-style `metadata.tools`. Do not log executable handlers or closures. +- Metrics must use only spec-approved keys. Do not emit Pipecat-specific metric keys such as `ttfb`, `ttfa`, `processing_time`, `turn_probability`, or `text_aggregation_time` in `metrics`. Map only the following when available: + - `tokens`, `prompt_tokens`, `completion_tokens` + - `time_to_first_token` for LLM streaming / first-token latency + - reasoning/cache/audio/image token keys if Pipecat exposes them with the same semantics + - `start` and `end` are handled by Braintrust span timing + Pipecat-specific timing/diagnostic values may be omitted or, for non-LLM framework spans only, placed in conservative metadata if they are useful and JSON-serializable. +- Inline media must be converted in-place to Braintrust attachments. For Pipecat this matters for `LLMContext.create_image_message()`, `LLMContext.create_audio_message()`, `input_audio` content parts, `TTSAudioRawFrame`, and any future generated media frames. + +## Proposed span shape + +Pipecat is an orchestration/framework integration. The integration should produce a spec-conformant agentic span tree while staying conservative about framework-only events. + +Recommended hierarchy: + +```text +pipecat_pipeline (task span, one per PipelineWorker run/session) + user_turn (task span, optional; one per turn) + stt_transcription (task span/event from TranscriptionFrame) + llm_response (llm span, one Pipecat model response) + (tool span for FunctionCallInProgress/Result) + tts_response (task span, text-to-speech operation) +``` + +LLM span requirements: + +- `span_attributes.type = "llm"` and name such as `"pipecat_llm_response"`. +- `input`: ordered OpenAI-style message array from the nearest `LLMContextFrame.context.get_messages()`. Preserve Pipecat's universal message dicts/lists directly when they already match the required shape; only materialize attachments and remove/skip clearly non-loggable runtime objects. Do not run broad recursive conversion or JSON round-trips. +- `output`: one OpenAI-style choice object: + - normal streamed text: `[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": aggregated_text}}]` + - model-requested tool calls: `finish_reason = "tool_calls"`, `message.content = null` or accompanying text, and `message.tool_calls` populated from `FunctionCallsStartedFrame` / `FunctionCallFromLLM`. +- `metadata`: `provider` and `model` when derivable from the Pipecat service/metrics frame; allowed request config fields only; `tools` and `tool_choice` from `LLMContext.tools` / `LLMContext.tool_choice` when present. Keep metadata values as normal Python dicts/lists/scalars; do not pre-stringify them for native Braintrust logging. +- `metrics`: only spec-approved metrics. Convert `LLMUsageMetricsData.value` to `tokens`, `prompt_tokens`, `completion_tokens`, plus cache/reasoning token keys when present. Convert LLM `TTFBMetricsData` to `time_to_first_token` only when it measures first streamed token/chunk latency. + +Tool span requirements: + +- `span_attributes.type = "tool"`. +- Span name should be the actual function/tool name, not a generic `function_tool`. +- `input`: arguments supplied by the model, as a dict/object when available. +- `output`: result from `FunctionCallResultFrame`. +- Correlate by `tool_call_id`. If a tool errors/cancels, log `error` and close the span. + +Task / media span requirements: + +- `pipecat_pipeline` and optional turn/STT/TTS spans use `span_attributes.type = "task"`. +- Pipeline `input` may include worker/pipeline metadata (`worker.name`, processor names, sample rates), but do not use an `llm` span for this framework metadata. +- `stt_transcription` output should be transcript text/JSON plus user/language metadata. +- `tts_response` input is text; output should either omit raw audio or contain generated audio as a Braintrust `Attachment` when audio capture is explicitly enabled. +- Do not log raw `TTSAudioRawFrame.audio` bytes inline. By default, record audio byte counts/sample rate/channel count as non-LLM task metadata only. If `capture_audio=True` is added, aggregate chunks into WAV and materialize an attachment in `output`, not metadata. + +### Attachment strategy + +Use attachments where the spec requires them, but keep capture conservative so the integration does not unexpectedly upload large real-time media streams. + +Initial implementation should: + +- Materialize inline media already present in logged LLM messages: + - `data:image/...;base64,...` in Pipecat `LLMContext` image content parts. + - `input_audio.data` from `LLMContext.create_audio_message()` and similar audio content parts. + - file/document data if Pipecat message content uses OpenAI-style `file.file_data`. +- Use the shared Braintrust attachment helpers (`_materialize_attachment`, `_materialize_chat_message_content_part`, `_normalize_chat_messages`) and pass `Attachment` objects in the logged `input`/`output`. Do not hand-build canonical `braintrust_attachment` dictionaries in the integration. +- Preserve remote URLs as URLs; do not fetch arbitrary remote media solely to create attachments. +- Preserve unsupported or failed-to-decode media values unchanged instead of dropping them. + +Initial implementation should not: + +- Attach every inbound `InputAudioRawFrame` / `AudioRawFrame`; continuous microphone streams are too large and noisy for default tracing. +- Attach `TTSAudioRawFrame` output by default. +- Add `attachments` lists in metadata or output. Attachments must live at the exact input/output leaf they replace. + +Follow-up / opt-in: + +- Add `capture_audio=True` only after the core integration is stable. When enabled, aggregate TTS output chunks per `TTSStartedFrame.context_id` / `TTSStoppedFrame.context_id`, encode a WAV, and place the resulting `Attachment` in `tts_response.output`. +- Consider a separate, bounded `capture_stt_audio=True` later for segmented STT tests or debugging, with size limits and no default continuous-stream capture. + +## Implementation plan + +### Package layout + +Create: + +- `py/src/braintrust/integrations/pipecat/__init__.py` +- `py/src/braintrust/integrations/pipecat/integration.py` +- `py/src/braintrust/integrations/pipecat/patchers.py` +- `py/src/braintrust/integrations/pipecat/tracing.py` +- `py/src/braintrust/integrations/pipecat/test_pipecat.py` + +Public API: + +```python +from braintrust.integrations.pipecat import setup_pipecat, wrap_pipeline_worker +``` + +### Patcher design + +Use one main patcher: + +- `PipelineWorkerInitPatcher` + - `target_module = "pipecat.pipeline.worker"` + - `target_path = "PipelineWorker.__init__"` + - wrapper mutates/extends the `observers` kwarg to include one `BraintrustPipecatObserver`, unless one is already present. + +Optional follow-up patchers: + +- `PipelineWorkerRunPatcher` only if observer lifecycle cannot reliably close root spans on all terminal paths. +- `PipelineTaskInitPatcher` only if we later decide to support Pipecat `1.0.0`-`1.2.x`. + +Manual wrapping: + +- `wrap_pipeline_worker(PipelineWorker)` should apply the worker init patcher to a provided class. +- A direct `BraintrustPipecatObserver` should also be exported for users who prefer explicit observer wiring without global patching. + +### Integration class + +```python +class PipecatIntegration(BaseIntegration): + name = "pipecat" + import_names = ("pipecat",) + distribution_names = ("pipecat-ai",) + min_version = "1.3.0" + patchers = (PipelineWorkerPatcher,) +``` + +### Auto instrumentation + +Implemented: + +- Export `PipecatIntegration` from `py/src/braintrust/integrations/__init__.py`. +- Add Pipecat to `braintrust.auto_instrument()` so users can call `braintrust.auto_instrument()` before constructing a `PipelineWorker`. +- Add `py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py` to verify import-order behavior in a subprocess. + +### Nox and dependency wiring + +Implemented: + +- `test-pipecat` dependency group with `websockets==15.0.1`; this is required because `pipecat-ai==1.3.0` imports `websockets` but does not install it transitively. +- `[tool.braintrust.matrix.pipecat-ai]` with `latest = "pipecat-ai==1.4.0"` and floor `"1.3.0" = "pipecat-ai==1.3.0"`. +- `[tool.braintrust.cassette-dirs]` entry for `pipecat = ["pipecat-ai"]` because the test suite is VCR-backed. +- `PIPECAT_VERSIONS = _get_matrix_versions("pipecat-ai")`. +- `test_pipecat(session, version)` nox session that skips on Python `<3.11`, skips Python `>=3.14` for Pipecat's current `onnxruntime` dependency, and runs `braintrust/integrations/pipecat/test_pipecat.py`. + +## Testing strategy + +### Primary tests: VCR-backed real Pipecat/OpenAI pipelines + +The implemented tests intentionally use VCR-backed real Pipecat/OpenAI pipelines rather than fabricated frames. This keeps the observer state machine tied to actual Pipecat service behavior and real provider payload shapes. + +Current coverage: + +1. `setup_pipecat()` injects a `BraintrustPipecatObserver` into newly constructed `PipelineWorker` objects. +2. Explicit `BraintrustPipecatObserver()` works without adding a duplicate injected observer. +3. Repeated `setup_pipecat()` and `wrap_pipeline_worker()` calls are idempotent. +4. A real `Pipeline([OpenAILLMService(...)])` receives an `LLMContextFrame`, calls OpenAI through Pipecat, and emits a `pipecat_pipeline` task span. +5. The real OpenAI/Pipecat response emits a spec-conformant `pipecat_llm_response` llm span with OpenAI-style message-array input, choices-array output, `metadata.provider`, `metadata.model`, and token metrics. +6. Metrics assertions verify only spec-approved metric keys are present, allowing Braintrust span timing keys (`start`, `end`) alongside token and first-token metrics. +7. Python `<3.11` nox sessions skip cleanly; Python `>=3.14` is skipped until Pipecat's `onnxruntime` dependency ships wheels. +8. `braintrust.auto_instrument()` is covered by a subprocess script that constructs and runs a real VCR-backed Pipecat/OpenAI pipeline. + +Cassette locations: + +- `py/src/braintrust/integrations/pipecat/cassettes/latest/` +- `py/src/braintrust/integrations/pipecat/cassettes/1.3.0/` + +Run locally: + +```bash +cd py +nox -s "test_pipecat(latest)" +nox -s "test_pipecat(1.3.0)" +OPENAI_API_KEY=sk-test-dummy-api-key-for-vcr-tests \ + nox -s "test_pipecat(latest)" -- -k test_setup_pipecat_traces_real_pipeline_frames +``` + +### Follow-up tests still worth adding + +The initial VCR suite does not yet cover every span shape described above. Useful follow-ups: + +- Real Pipecat tool-call behavior that creates child `tool` spans from actual `FunctionCallsStartedFrame` / `FunctionCallInProgressFrame` / `FunctionCallResultFrame` sequences. +- Real transcription and TTS service behavior, preferably with narrow VCR cassettes when provider traffic is involved. +- Multimodal `LLMContext` attachment materialization with real provider-compatible payloads. +- Terminal-frame variants (`StopFrame`, `CancelFrame`) and cancellation cleanup. + +## Additional refinements before implementation + +- **Split initial scope from follow-up scope.** Initial implementation should cover worker/session task spans, LLM response spans, tool spans, transcript spans, TTS text/audio metadata, and spec-approved token metrics. Defer runner/bus spans, OTel bridging, realtime-provider-specific event lifecycles, and opt-in audio attachments. +- **Make observer configuration explicit but small.** Consider `setup_pipecat(capture_audio=False, trace_turns=True)` and `BraintrustPipecatObserver(capture_audio=False, trace_turns=True)`. Avoid many knobs until real users need them. +- **Define state correlation carefully.** The observer will need per-worker/per-pipeline state for active pipeline span, active turn span, active LLM span, active TTS span by `context_id`, and active tools by `tool_call_id`. Keep this state on the observer instance, not module globals, so concurrent Pipecat workers do not cross-contaminate traces. +- **Prefer best-effort provider/model attribution.** Use Pipecat metrics data (`processor`, `model`) and processor class names to populate `metadata.model` / `metadata.provider` when clear. If provider cannot be determined, omit the child `llm` span or use a conservative provider value only if the spec allows it; do not invent provider names that would affect pricing semantics. +- **Preserve user span context.** If a Pipecat worker is created/run inside an existing Braintrust span, attach `pipecat_pipeline` beneath that current span. Child spans should attach through exported parent contexts rather than relying on global current-span state across observer queue tasks. +- **Do not make Braintrust logging affect the pipeline.** Observer code should never change frame flow, return values, or Pipecat error behavior. Conversion failures should preserve original values or omit optional fields, not raise into Pipecat observer processing. +- **Test minimal shaping.** Add tests that pass already-valid message dicts and assert they are preserved without stringification or JSON round-trips, while inline media leaves are materialized in place. + +## Risks and open questions + +- **Long-running sessions:** Voice agents can run indefinitely. The root span must close on all terminal frames and also during worker cancellation/cleanup. +- **Metric spec compliance:** Pipecat exposes useful metrics (`ttfb`, `ttfa`, processing time, turn prediction), but the Braintrust instrumentation guide only permits a fixed metric key set. The integration must map only semantically compatible values and omit or demote the rest. +- **Duplicate provider spans:** Pipecat framework LLM spans may overlap with direct provider integrations if users also instrument OpenAI/Anthropic/etc. Do not suppress provider leaf spans implicitly; document the possible duplication and keep Pipecat span structure spec-conformant. +- **Existing workers:** `setup_pipecat()` cannot inject into workers constructed before setup unless we add a `run()` patcher. Document setup-before-construction as the expected path. +- **Audio attachments:** Useful but potentially large. Keep raw audio capture off by default. +- **Multi-worker/bus support:** Initial scope should trace each `PipelineWorker`; later work can add runner/bus-level spans if users need cross-worker traces. +- **Pipecat 1.5.0+ changes:** Main already contains a deprecation note for `WorkerRunner(loop=...)` in 1.5.0, so keep patch targets focused on `PipelineWorker.__init__` and observer frames rather than runner internals. diff --git a/py/noxfile.py b/py/noxfile.py index 6bc831ed..eaacd013 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -373,6 +373,22 @@ def test_livekit_agents(session, version): _run_tests(session, f"{INTEGRATION_DIR}/livekit_agents/test_livekit_agents.py", version=version, env=env) +PIPECAT_VERSIONS = _get_matrix_versions("pipecat-ai") + + +@nox.session() +@nox.parametrize("version", PIPECAT_VERSIONS, ids=PIPECAT_VERSIONS) +def test_pipecat(session, version): + if sys.version_info < (3, 11): + session.skip("Pipecat AI 1.x requires Python 3.11+") + if sys.version_info >= (3, 14): + session.skip("Pipecat AI's onnxruntime dependency does not ship Python 3.14 wheels") + _install_test_deps(session) + _install_group_locked(session, "test-pipecat") + _install_matrix_dep(session, "pipecat-ai", version) + _run_tests(session, f"{INTEGRATION_DIR}/pipecat/test_pipecat.py", version=version) + + STRANDS_VERSIONS = _get_matrix_versions("strands-agents") diff --git a/py/pyproject.toml b/py/pyproject.toml index d693372c..57001df5 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -166,6 +166,12 @@ test-livekit-agents = [ "opentelemetry-sdk<1.39", ] +test-pipecat = [ + {include-group = "test"}, + # pipecat-ai 1.3.0 imports websockets but does not install it transitively. + "websockets==15.0.1", +] + test-crewai = [ {include-group = "test"}, # CrewAI's no-network smoke test forces the LiteLLM fallback path via @@ -338,6 +344,10 @@ latest = "litellm==1.90.0" latest = "livekit-agents==1.6.4" "1.3.1" = "livekit-agents==1.3.1" +[tool.braintrust.matrix.pipecat-ai] +latest = "pipecat-ai==1.4.0" +"1.3.0" = "pipecat-ai==1.3.0" + [tool.braintrust.matrix.claude-agent-sdk] latest = "claude-agent-sdk==0.2.110" "0.1.10" = "claude-agent-sdk==0.1.10" @@ -485,6 +495,7 @@ mistral = ["mistralai"] openai = ["openai"] openai_agents = ["openai-agents"] openrouter = ["openrouter"] +pipecat = ["pipecat-ai"] pydantic_ai = ["pydantic-ai-integration", "pydantic-ai-wrap-openai"] strands = ["strands-agents"] @@ -510,5 +521,6 @@ huggingface-hub = "huggingface_hub" openai = "openai" openai-agents = "agents" openrouter = "openrouter" +pipecat-ai = "pipecat" strands-agents = "strands" temporalio = "temporalio" diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index a16aa384..b873fe6d 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -29,6 +29,7 @@ OpenAIAgentsIntegration, OpenAIIntegration, OpenRouterIntegration, + PipecatIntegration, PydanticAIIntegration, StrandsIntegration, TemporalIntegration, @@ -78,6 +79,7 @@ def auto_instrument( strands: bool = True, temporal: bool = True, livekit_agents: bool = True, + pipecat: bool = True, ) -> dict[str, bool]: """ Auto-instrument supported AI/ML libraries for Braintrust tracing. @@ -113,6 +115,7 @@ def auto_instrument( strands: Enable Strands Agents instrumentation (default: True) temporal: Enable Temporal instrumentation (default: True) livekit_agents: Enable LiveKit Agents instrumentation (default: True) + pipecat: Enable Pipecat AI instrumentation (default: True) Returns: Dict mapping integration name to whether it was successfully instrumented. @@ -208,6 +211,8 @@ def auto_instrument( results["temporal"] = _instrument_integration(TemporalIntegration) if livekit_agents: results["livekit_agents"] = _instrument_integration(LiveKitAgentsIntegration) + if pipecat: + results["pipecat"] = _instrument_integration(PipecatIntegration) return results diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index 4cb63923..c9b69d9a 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -19,6 +19,7 @@ from .openai import OpenAIIntegration from .openai_agents import OpenAIAgentsIntegration from .openrouter import OpenRouterIntegration +from .pipecat import PipecatIntegration from .pydantic_ai import PydanticAIIntegration from .strands import StrandsIntegration from .temporal import TemporalIntegration @@ -46,6 +47,7 @@ "OpenAIIntegration", "OpenAIAgentsIntegration", "OpenRouterIntegration", + "PipecatIntegration", "PydanticAIIntegration", "StrandsIntegration", "TemporalIntegration", diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py new file mode 100644 index 00000000..e32d8d54 --- /dev/null +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py @@ -0,0 +1,102 @@ +import asyncio +import importlib +import inspect +import os +import tempfile +from pathlib import Path + +from braintrust.auto import auto_instrument +from braintrust.integrations.test_utils import autoinstrument_test_context + + +def _ensure_nltk_punkt_tab(): + data_dir = Path(tempfile.gettempdir()) / "braintrust-pipecat-nltk-data" + punkt_tab = data_dir / "tokenizers" / "punkt_tab" + punkt_tab.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("NLTK_DATA", str(data_dir)) + + +def _import(path): + _ensure_nltk_punkt_tab() + module_name, attr = path.rsplit(".", 1) + return getattr(importlib.import_module(module_name), attr) + + +def _worker_kwargs(**overrides): + PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") + signature = inspect.signature(PipelineWorker) + kwargs = {"idle_timeout_secs": None} + for name, value in { + "enable_turn_tracking": False, + "enable_rtvi": False, + "check_dangling_tasks": False, + }.items(): + if name in signature.parameters: + kwargs[name] = value + kwargs.update(overrides) + return kwargs + + +def _runner_kwargs(**overrides): + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") + signature = inspect.signature(WorkerRunner) + kwargs = {"handle_sigint": False} + if "check_dangling_tasks" in signature.parameters: + kwargs["check_dangling_tasks"] = False + kwargs.update(overrides) + return kwargs + + +async def main(): + with autoinstrument_test_context("test_auto_pipecat", integration="pipecat") as memory_logger: + _ensure_nltk_punkt_tab() + results = auto_instrument() + assert results.get("pipecat") is True + + EndFrame = _import("pipecat.frames.frames.EndFrame") + LLMContextFrame = _import("pipecat.frames.frames.LLMContextFrame") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + PipelineParams = _import("pipecat.pipeline.worker.PipelineParams") + PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") + LLMContext = _import("pipecat.processors.aggregators.llm_context.LLMContext") + OpenAILLMService = _import("pipecat.services.openai.llm.OpenAILLMService") + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") + + llm = OpenAILLMService( + api_key=os.environ["OPENAI_API_KEY"], + settings=OpenAILLMService.Settings( + model="gpt-4o-mini", + temperature=0.0, + max_completion_tokens=20, + ), + ) + worker = PipelineWorker( + Pipeline([llm]), + **_worker_kwargs( + name="bt-auto-pipecat-worker", + params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), + ), + ) + context = LLMContext( + messages=[ + {"role": "developer", "content": "Answer with exactly the requested text and no punctuation."}, + {"role": "user", "content": "Say: braintrust auto pipecat"}, + ] + ) + + @worker.event_handler("on_pipeline_started") + async def on_pipeline_started(_worker, _frame): + await worker.queue_frames([LLMContextFrame(context), EndFrame()]) + + runner = WorkerRunner(**_runner_kwargs()) + await runner.add_workers(worker) + await asyncio.wait_for(runner.run(), timeout=20) + + logs = memory_logger.pop() + names = {log.get("span_attributes", {}).get("name") for log in logs} + assert "pipecat_pipeline" in names + assert "pipecat_llm_response" in names + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/py/src/braintrust/integrations/pipecat/__init__.py b/py/src/braintrust/integrations/pipecat/__init__.py new file mode 100644 index 00000000..ed4e6bc7 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/__init__.py @@ -0,0 +1,38 @@ +"""Braintrust integration for Pipecat AI.""" + +from braintrust.logger import NOOP_SPAN, current_span, init_logger + +from .integration import PipecatIntegration +from .patchers import set_default_observer_options, wrap_pipeline_worker +from .tracing import BraintrustPipecatObserver + + +__all__ = [ + "BraintrustPipecatObserver", + "PipecatIntegration", + "setup_pipecat", + "wrap_pipeline_worker", +] + + +def setup_pipecat( + api_key: str | None = None, + project_id: str | None = None, + project_name: str | None = None, + *, + capture_audio: bool = False, + trace_turns: bool = True, +) -> bool: + """Set up Braintrust tracing for Pipecat ``PipelineWorker`` instances. + + The setup hook patches ``PipelineWorker.__init__`` so newly constructed + workers receive a ``BraintrustPipecatObserver`` unless one was already + provided explicitly. + """ + set_default_observer_options(capture_audio=capture_audio, trace_turns=trace_turns) + + span = current_span() + if span == NOOP_SPAN: + init_logger(project=project_name, api_key=api_key, project_id=project_id) + + return PipecatIntegration.setup() diff --git a/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_instrument_pipecat_subprocess.yaml b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_instrument_pipecat_subprocess.yaml new file mode 100644 index 00000000..ab370bd5 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_instrument_pipecat_subprocess.yaml @@ -0,0 +1,2 @@ +interactions: [] +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_pipecat.yaml b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_pipecat.yaml new file mode 100644 index 00000000..76b370b9 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_auto_pipecat.yaml @@ -0,0 +1,131 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Answer with exactly the requested + text and no punctuation."},{"role":"user","content":"Say: braintrust auto pipecat"}],"model":"gpt-4o-mini","max_completion_tokens":20,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '284' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.44.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.44.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mBvPhUhAJ"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"brain"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"f6iO50"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"trust"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1OZQen"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":" + auto"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"nNikol"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":" + pi"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"xXSuQcBG"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"pec"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"fFb1am2g"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"at"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Q6pMNbfZz"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"np4gz"} + + + data: {"id":"chatcmpl-DxeZpXNoUnRnT8t7XsNZZ5V83gre4","object":"chat.completion.chunk","created":1783109701,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[],"usage":{"prompt_tokens":29,"completion_tokens":6,"total_tokens":35,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"oyfaDmQniIW"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1588f52ad9ea22e-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Fri, 03 Jul 2026 20:15:02 GMT + openai-organization: + - braintrust-data + openai-processing-ms: + - '706' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=1Q17IKR.MBtQXtkQ1S5ly5x3ajkpcTkBBxgFtfRHkf4-1783109701.5507047-1.0.1.1-.CetNrjD1s17f4gfSC5MYP5bHVCA5SUrLsH5suopqdkp.vCr0k9we7_F2wjwQYKSt4LZ4Ca5n2SCUp_csLqra2jfPYwmFjJv53cWGbRnNI3vg6wHBiwdrRhBNAa9NoAE; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, + 03 Jul 2026 20:45:02 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999975' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d7265fcbe8f444b8968fbebc9eedf37c + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml new file mode 100644 index 00000000..ab370bd5 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml @@ -0,0 +1,2 @@ +interactions: [] +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_pipecat_traces_real_pipeline_frames.yaml b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_pipecat_traces_real_pipeline_frames.yaml new file mode 100644 index 00000000..61349abf --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/1.3.0/test_setup_pipecat_traces_real_pipeline_frames.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Answer with exactly the requested + text and no punctuation."},{"role":"user","content":"Say: braintrust pipecat + integration"}],"model":"gpt-4o-mini","max_completion_tokens":20,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '291' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.44.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.44.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"DisLDiYyS"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"brain"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"fdchpY"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"trust"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"qlDymW"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":" + pi"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"O5RavUFQ"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"pec"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"gstZAYv7"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"at"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"RYWV8IE1V"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":" + integration"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ZiHa5L8gYucQH3m"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"ZMvqN"} + + + data: {"id":"chatcmpl-DxeZnsj8obHzCvESdmdCOJdloJrrQ","object":"chat.completion.chunk","created":1783109699,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[],"usage":{"prompt_tokens":29,"completion_tokens":6,"total_tokens":35,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"wreEVBMklwx"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1588f435cf28eb6-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Fri, 03 Jul 2026 20:14:59 GMT + openai-organization: + - braintrust-data + openai-processing-ms: + - '337' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=Qn3v3zix179SI4EsOyQMZkhURzFAjHFzmZ_yUoidoi4-1783109699.0914087-1.0.1.1-vTCWXqVH9zco_2nXTGMpFKxDiYC0xG6HtU7Omh6whEMbupFlpynNuzft0VVRE_6O.GG_gm3lvLbXSELTcFotmarmH1DcsgE0Pesw4mLYgZzokXmUopgjemA556LUCk1s; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, + 03 Jul 2026 20:44:59 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999972' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3921db5e4be7481f97f14f729b277030 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_instrument_pipecat_subprocess.yaml b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_instrument_pipecat_subprocess.yaml new file mode 100644 index 00000000..ab370bd5 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_instrument_pipecat_subprocess.yaml @@ -0,0 +1,2 @@ +interactions: [] +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_pipecat.yaml b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_pipecat.yaml new file mode 100644 index 00000000..06acba3c --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_auto_pipecat.yaml @@ -0,0 +1,131 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Answer with exactly the requested + text and no punctuation."},{"role":"user","content":"Say: braintrust auto pipecat"}],"model":"gpt-4o-mini","max_completion_tokens":20,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '284' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.44.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.44.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"42d7fvH7u"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"brain"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"64Zj6i"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"trust"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"U1bQfN"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":" + auto"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"QIM4ua"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":" + pi"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ZAf15MLs"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"pec"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"K9al2a8C"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{"content":"at"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4XLXIuocC"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Zl7Nu"} + + + data: {"id":"chatcmpl-DxeYGwwPYBD3m8rFh8623xfsYk3FM","object":"chat.completion.chunk","created":1783109604,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_4507413d49","choices":[],"usage":{"prompt_tokens":29,"completion_tokens":6,"total_tokens":35,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"nX4CsdvqDpm"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a1588cf4bd5daabc-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Fri, 03 Jul 2026 20:13:25 GMT + openai-organization: + - braintrust-data + openai-processing-ms: + - '249' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=PiEPmcKQbcxLyFRhC8lGlp.2UbnhW1nGr6zabRgztJc-1783109604.601541-1.0.1.1-YcYtXpMd_V2rPuhuVO1zFZP01N2G9oY3UF0vp6ZCxCuTYxdynDYCXt0QRZ.Ofd31M3gOkzJBC8pYCllqEyvyI6qehzzYiNCc5g9RwHbwtIjEWb.8cY2xrETXbPfFKuRO; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, + 03 Jul 2026 20:43:25 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999975' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_86f56b2f994d493385ffd3d5f6634453 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml new file mode 100644 index 00000000..ab370bd5 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_and_wrap_pipeline_worker_are_idempotent.yaml @@ -0,0 +1,2 @@ +interactions: [] +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_pipecat_traces_real_pipeline_frames.yaml b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_pipecat_traces_real_pipeline_frames.yaml new file mode 100644 index 00000000..47bd055d --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_setup_pipecat_traces_real_pipeline_frames.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"messages":[{"role":"developer","content":"Answer with exactly the requested + text and no punctuation."},{"role":"user","content":"Say: braintrust pipecat + integration"}],"model":"gpt-4o-mini","max_completion_tokens":20,"stream":true,"stream_options":{"include_usage":true},"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '291' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.44.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.44.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.12 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"6zQEYqNcX"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"brain"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"qX7NwI"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"trust"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ytHbN8"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":" + pi"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"KYDX2Fy8"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"pec"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"YObp5ucS"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":"at"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"E84Daht6o"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{"content":" + integration"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"cNd8XYxJrg6oUSY"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Q0KoC"} + + + data: {"id":"chatcmpl-DxeVKBBtz3Bb13BNxFdFsvfzfpu8j","object":"chat.completion.chunk","created":1783109422,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_a151a364ab","choices":[],"usage":{"prompt_tokens":29,"completion_tokens":6,"total_tokens":35,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"MQdx366dBfc"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a158887cacf1ac58-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Fri, 03 Jul 2026 20:10:22 GMT + openai-organization: + - braintrust-data + openai-processing-ms: + - '330' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + server: + - cloudflare + set-cookie: + - __cf_bm=UI5TxCZl.0C6CHLgbxbZrYJzZazAYNN1HgzJHKyw1Ms-1783109421.548962-1.0.1.1-.I3EynjA0LzfvVUtjVA6EvyUfr4gmO8XotUySmUENoeJpEz3maeKXzN_uDe.yrGg3QwfSnH39iXc1X1mDvdAPMkIrvMkozKNyWTnihGGiImMEbN_1YcDap29av2SJJmc; + HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Fri, + 03 Jul 2026 20:40:22 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999972' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4de2c0388e0f4f2a80896e8a8b0a4f7e + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pipecat/integration.py b/py/src/braintrust/integrations/pipecat/integration.py new file mode 100644 index 00000000..7cbde1b5 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/integration.py @@ -0,0 +1,15 @@ +"""Pipecat integration.""" + +from braintrust.integrations.base import BaseIntegration + +from .patchers import PipelineWorkerPatcher + + +class PipecatIntegration(BaseIntegration): + """Braintrust instrumentation for Pipecat AI pipelines.""" + + name = "pipecat" + import_names = ("pipecat",) + distribution_names = ("pipecat-ai",) + min_version = "1.3.0" + patchers = (PipelineWorkerPatcher,) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py new file mode 100644 index 00000000..b87f833d --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -0,0 +1,51 @@ +"""Pipecat integration patchers.""" + +from typing import Any + +from braintrust.integrations.base import FunctionWrapperPatcher + +from .tracing import BraintrustPipecatObserver + + +_DEFAULT_OBSERVER_OPTIONS: dict[str, Any] = {"capture_audio": False, "trace_turns": True} + + +def set_default_observer_options(**options: Any) -> None: + """Set options used when global patching injects a Braintrust observer.""" + _DEFAULT_OBSERVER_OPTIONS.update({key: value for key, value in options.items() if value is not None}) + + +def _has_braintrust_observer(observers: list[Any]) -> bool: + return any(isinstance(observer, BraintrustPipecatObserver) for observer in observers) + + +def _with_braintrust_observer(observers: Any) -> list[Any]: + ret = list(observers or []) + if not _has_braintrust_observer(ret): + ret.append(BraintrustPipecatObserver(**_DEFAULT_OBSERVER_OPTIONS)) + return ret + + +def traced_pipeline_worker_init(wrapped: Any, _instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + """Inject the Braintrust Pipecat observer into PipelineWorker construction.""" + kwargs = dict(kwargs) + kwargs["observers"] = _with_braintrust_observer(kwargs.get("observers")) + return wrapped(*args, **kwargs) + + +class PipelineWorkerInitPatcher(FunctionWrapperPatcher): + """Patch ``PipelineWorker.__init__`` to add a Braintrust observer.""" + + name = "pipeline_worker_init" + target_module = "pipecat.pipeline.worker" + target_path = "PipelineWorker.__init__" + wrapper = staticmethod(traced_pipeline_worker_init) + + +# Alias used by the implementation plan and public helper. +PipelineWorkerPatcher = PipelineWorkerInitPatcher + + +def wrap_pipeline_worker(PipelineWorker: Any) -> Any: + """Instrument a Pipecat ``PipelineWorker`` class directly.""" + return PipelineWorkerInitPatcher.wrap_target(PipelineWorker) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py new file mode 100644 index 00000000..3230aa07 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -0,0 +1,180 @@ +import asyncio +import importlib +import inspect +import os +import tempfile +from pathlib import Path + +import pytest +from braintrust import logger +from braintrust.integrations.pipecat import ( + BraintrustPipecatObserver, + PipecatIntegration, + setup_pipecat, + wrap_pipeline_worker, +) +from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.test_helpers import init_test_logger + + +@pytest.fixture +def memory_logger(): + init_test_logger("test-project-pipecat-py-tracing") + with logger._internal_with_memory_background_logger() as bgl: + yield bgl + + +def _ensure_nltk_punkt_tab(): + data_dir = Path(tempfile.gettempdir()) / "braintrust-pipecat-nltk-data" + punkt_tab = data_dir / "tokenizers" / "punkt_tab" + punkt_tab.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("NLTK_DATA", str(data_dir)) + + +def _import(path): + _ensure_nltk_punkt_tab() + module_name, attr = path.rsplit(".", 1) + return getattr(importlib.import_module(module_name), attr) + + +def _span_name(log): + return log.get("span_attributes", {}).get("name") + + +def _span_type(log): + return log.get("span_attributes", {}).get("type") + + +def _spans_named(logs, name): + return [log for log in logs if _span_name(log) == name] + + +def _single_span(logs, name): + matches = _spans_named(logs, name) + assert len(matches) == 1, (name, matches) + return matches[0] + + +def _pipeline_worker_kwargs(**overrides): + PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") + signature = inspect.signature(PipelineWorker) + kwargs = {"idle_timeout_secs": None} + for name, value in { + "enable_turn_tracking": False, + "enable_rtvi": False, + "check_dangling_tasks": False, + }.items(): + if name in signature.parameters: + kwargs[name] = value + kwargs.update(overrides) + return kwargs + + +def _make_worker(pipeline, **overrides): + PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") + return PipelineWorker(pipeline, **_pipeline_worker_kwargs(**overrides)) + + +def _worker_runner_kwargs(**overrides): + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") + signature = inspect.signature(WorkerRunner) + kwargs = {"handle_sigint": False} + if "check_dangling_tasks" in signature.parameters: + kwargs["check_dangling_tasks"] = False + kwargs.update(overrides) + return kwargs + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_setup_pipecat_traces_real_pipeline_frames(memory_logger): + EndFrame = _import("pipecat.frames.frames.EndFrame") + LLMContextFrame = _import("pipecat.frames.frames.LLMContextFrame") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + LLMContext = _import("pipecat.processors.aggregators.llm_context.LLMContext") + OpenAILLMService = _import("pipecat.services.openai.llm.OpenAILLMService") + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") + PipelineParams = _import("pipecat.pipeline.worker.PipelineParams") + + assert setup_pipecat(project_name="test-project-pipecat-py-tracing") + init_test_logger("test-project-pipecat-py-tracing") + llm = OpenAILLMService( + api_key=os.environ["OPENAI_API_KEY"], + settings=OpenAILLMService.Settings( + model="gpt-4o-mini", + temperature=0.0, + max_completion_tokens=20, + ), + ) + worker = _make_worker( + Pipeline([llm]), + name="bt-pipecat-test-worker", + params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), + ) + context = LLMContext( + messages=[ + {"role": "developer", "content": "Answer with exactly the requested text and no punctuation."}, + {"role": "user", "content": "Say: braintrust pipecat integration"}, + ] + ) + + @worker.event_handler("on_pipeline_started") + async def on_pipeline_started(_worker, _frame): + await worker.queue_frames([LLMContextFrame(context), EndFrame()]) + + runner = WorkerRunner(**_worker_runner_kwargs()) + await runner.add_workers(worker) + await asyncio.wait_for(runner.run(), timeout=20) + + logs = memory_logger.pop() + pipeline_span = _single_span(logs, "pipecat_pipeline") + assert _span_type(pipeline_span) == "task" + assert pipeline_span.get("metrics", {}).get("end") is not None + + llm_span = _single_span(logs, "pipecat_llm_response") + assert _span_type(llm_span) == "llm" + assert llm_span["input"] == context.messages + assert llm_span["output"][0]["finish_reason"] == "stop" + assert "braintrust pipecat integration" in llm_span["output"][0]["message"]["content"].lower() + assert llm_span["metadata"]["provider"] == "openai" + assert llm_span["metadata"]["model"] == "gpt-4o-mini" + assert set(llm_span.get("metrics", {})) <= { + "time_to_first_token", + "prompt_tokens", + "completion_tokens", + "tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", + "start", + "end", + } + assert llm_span["metrics"]["prompt_tokens"] > 0 + assert llm_span["metrics"]["completion_tokens"] > 0 + assert llm_span["metrics"]["tokens"] >= llm_span["metrics"]["completion_tokens"] + + +@pytest.mark.vcr +def test_setup_and_wrap_pipeline_worker_are_idempotent(): + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") + IdentityFilter = _import("pipecat.processors.filters.identity_filter.IdentityFilter") + + assert PipecatIntegration.min_version == "1.3.0" + assert setup_pipecat(project_name="test-project-pipecat-py-tracing") + assert setup_pipecat(project_name="test-project-pipecat-py-tracing") + assert wrap_pipeline_worker(PipelineWorker) is PipelineWorker + + explicit_observer = BraintrustPipecatObserver() + worker = _make_worker(Pipeline([IdentityFilter()]), observers=[explicit_observer]) + worker_observer = getattr(worker, "_observer") + observers = getattr(worker_observer, "_observers") + braintrust_observers = [observer for observer in observers if isinstance(observer, BraintrustPipecatObserver)] + assert braintrust_observers == [explicit_observer] + + +@pytest.mark.vcr +@pytest.mark.skipif(__import__("sys").version_info < (3, 11), reason="Pipecat AI 1.x requires Python 3.11+") +def test_auto_instrument_pipecat_subprocess(): + pytest.importorskip("pipecat") + verify_autoinstrument_script("test_auto_pipecat.py") diff --git a/py/src/braintrust/integrations/pipecat/tracing.py b/py/src/braintrust/integrations/pipecat/tracing.py new file mode 100644 index 00000000..1771e904 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/tracing.py @@ -0,0 +1,564 @@ +"""Observer-based tracing for Pipecat pipelines.""" + +import json +from typing import Any + +from braintrust.integrations.utils import _is_not_given, _normalize_chat_messages +from braintrust.logger import NOOP_SPAN, SpanTypeAttribute, current_span, start_span + + +try: + from pipecat.observers.base_observer import BaseObserver +except ImportError: # pragma: no cover - exercised when Pipecat is not installed. + + class BaseObserver: # type: ignore[no-redef] + """Fallback base so this module is importable without Pipecat.""" + + def __init__(self, **_kwargs: Any) -> None: + pass + + async def cleanup(self) -> None: + pass + + +_TERMINAL_FRAME_TYPES = {"EndFrame", "StopFrame", "CancelFrame"} +_SPEC_METRIC_NAMES = { + "tokens", + "prompt_tokens", + "completion_tokens", + "time_to_first_token", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", +} +_ALLOWED_LLM_METADATA_FIELDS = { + "model", + "provider", + "frequency_penalty", + "presence_penalty", + "seed", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "tools", + "tool_choice", +} + + +class BraintrustPipecatObserver(BaseObserver): + """Pipecat observer that emits Braintrust task, llm, and tool spans.""" + + def __init__(self, *, capture_audio: bool = False, trace_turns: bool = True, **kwargs: Any) -> None: + super().__init__(**kwargs) + self.capture_audio = capture_audio + self.trace_turns = trace_turns + self._parent = _current_parent_export() + self._pipeline_span: Any | None = None + self._pipeline_parent: str | None = None + self._seen_frame_ids: set[int] = set() + self._latest_llm_input: Any = None + self._latest_llm_metadata: dict[str, Any] = {} + self._llm_span: Any | None = None + self._llm_parent: str | None = None + self._llm_text_parts: list[str] = [] + self._llm_metrics: dict[str, Any] = {} + self._llm_metadata: dict[str, Any] = {} + self._llm_tool_calls: list[dict[str, Any]] = [] + self._tool_call_parents: dict[str, str] = {} + self._tool_spans: dict[str, Any] = {} + self._tts_spans: dict[str, Any] = {} + self._tts_default_span: Any | None = None + + async def on_pipeline_started(self) -> None: + self._ensure_pipeline_span() + + async def on_process_frame(self, data: Any) -> None: + await self._handle_frame(getattr(data, "frame", None), processor=getattr(data, "processor", None)) + + async def on_push_frame(self, data: Any) -> None: + await self._handle_frame(getattr(data, "frame", None), processor=getattr(data, "source", None)) + + async def cleanup(self) -> None: + self._close_all_open_spans() + await super().cleanup() + + async def _handle_frame(self, frame: Any, *, processor: Any = None) -> None: + if frame is None: + return + frame_type = type(frame).__name__ + if frame_type in _TERMINAL_FRAME_TYPES and not _is_pipeline_sink_processor(processor): + return + + frame_id = getattr(frame, "id", None) + if isinstance(frame_id, int): + if frame_id in self._seen_frame_ids: + return + self._seen_frame_ids.add(frame_id) + if frame_type == "StartFrame": + self._ensure_pipeline_span(frame=frame, processor=processor) + elif frame_type == "LLMContextFrame": + self._capture_llm_context(frame, processor) + elif frame_type == "LLMFullResponseStartFrame": + self._start_llm_span(processor) + elif frame_type == "LLMTextFrame": + if self._llm_span is not None: + self._llm_text_parts.append(getattr(frame, "text", "")) + elif frame_type == "FunctionCallsStartedFrame": + self._capture_llm_tool_calls(frame) + elif frame_type == "LLMFullResponseEndFrame": + self._end_llm_span() + elif frame_type == "FunctionCallInProgressFrame": + self._start_tool_span(frame) + elif frame_type == "FunctionCallResultFrame": + self._end_tool_span(frame) + elif frame_type == "FunctionCallCancelFrame": + self._cancel_tool_span(frame) + elif frame_type == "TranscriptionFrame": + self._log_transcription(frame) + elif frame_type == "TTSStartedFrame": + self._start_tts_span(frame) + elif frame_type == "TTSTextFrame": + self._append_tts_text(frame) + elif frame_type == "TTSAudioRawFrame": + self._log_tts_audio_metadata(frame) + elif frame_type == "TTSStoppedFrame": + self._end_tts_span(frame) + elif frame_type == "MetricsFrame": + self._capture_metrics(frame, processor) + elif frame_type == "ErrorFrame": + self._log_error_frame(frame) + elif frame_type in _TERMINAL_FRAME_TYPES: + self._end_pipeline_span(frame) + + def _ensure_pipeline_span(self, *, frame: Any | None = None, processor: Any = None) -> None: + if self._pipeline_span is not None: + return + input_payload: dict[str, Any] = {} + metadata: dict[str, Any] = {"framework": "pipecat"} + if frame is not None: + input_payload.update( + { + "audio_in_sample_rate": getattr(frame, "audio_in_sample_rate", None), + "audio_out_sample_rate": getattr(frame, "audio_out_sample_rate", None), + "enable_metrics": getattr(frame, "enable_metrics", None), + "enable_usage_metrics": getattr(frame, "enable_usage_metrics", None), + } + ) + metadata.update(getattr(frame, "metadata", {}) or {}) + processor_name = _processor_name(processor) + if processor_name: + metadata["processor"] = processor_name + input_payload = {k: v for k, v in input_payload.items() if v is not None} + self._pipeline_span = start_span( + name="pipecat_pipeline", + type=SpanTypeAttribute.TASK, + input=input_payload or None, + parent=self._parent, + set_current=False, + ) + self._pipeline_parent = self._pipeline_span.export() + if metadata: + self._pipeline_span.log(metadata=metadata) + + def _capture_llm_context(self, frame: Any, processor: Any) -> None: + self._ensure_pipeline_span(processor=processor) + context = getattr(frame, "context", None) + messages = _messages_from_context(context) + self._latest_llm_input = _normalize_chat_messages(messages) + metadata = _metadata_from_context(context) + metadata.update(_metadata_from_processor(processor)) + self._latest_llm_metadata = _filter_llm_metadata(metadata) + + def _start_llm_span(self, processor: Any) -> None: + self._ensure_pipeline_span(processor=processor) + if self._llm_span is not None: + return + metadata = {**self._latest_llm_metadata, **_metadata_from_processor(processor)} + self._llm_metadata = _filter_llm_metadata(metadata) + self._llm_text_parts = [] + self._llm_metrics = {} + self._llm_tool_calls = [] + self._llm_span = start_span( + name="pipecat_llm_response", + type=SpanTypeAttribute.LLM, + input=self._latest_llm_input, + metadata=self._llm_metadata or None, + parent=self._pipeline_parent, + set_current=False, + ) + self._llm_parent = self._llm_span.export() + + def _capture_llm_tool_calls(self, frame: Any) -> None: + calls = getattr(frame, "function_calls", None) or [] + for call in calls: + tool_call = _tool_call_from_pipecat_call(call) + if tool_call: + self._llm_tool_calls.append(tool_call) + tool_id = tool_call.get("id") + if isinstance(tool_id, str) and self._llm_parent: + self._tool_call_parents[tool_id] = self._llm_parent + + def _end_llm_span(self) -> None: + if self._llm_span is None: + return + content = "".join(part for part in self._llm_text_parts if isinstance(part, str)) + message: dict[str, Any] = {"role": "assistant", "content": None if self._llm_tool_calls else content} + if self._llm_tool_calls: + if content: + message["content"] = content + message["tool_calls"] = self._llm_tool_calls + output = [ + { + "index": 0, + "finish_reason": "tool_calls" if self._llm_tool_calls else "stop", + "message": message, + } + ] + event: dict[str, Any] = {"output": output} + if self._llm_metrics: + event["metrics"] = {k: v for k, v in self._llm_metrics.items() if k in _SPEC_METRIC_NAMES} + metadata = _filter_llm_metadata(self._llm_metadata) + if metadata: + event["metadata"] = metadata + self._llm_span.log(**event) + self._llm_span.end() + self._llm_span = None + self._llm_parent = None + self._llm_text_parts = [] + self._llm_metrics = {} + self._llm_metadata = {} + self._llm_tool_calls = [] + + def _start_tool_span(self, frame: Any) -> None: + self._ensure_pipeline_span() + tool_call_id = getattr(frame, "tool_call_id", None) + parent = self._tool_call_parents.get(tool_call_id) or self._llm_parent or self._pipeline_parent + name = getattr(frame, "function_name", None) or "pipecat_tool" + span = start_span( + name=name, + type=SpanTypeAttribute.TOOL, + input=_parse_jsonish(getattr(frame, "arguments", None)), + parent=parent, + set_current=False, + ) + if isinstance(tool_call_id, str): + self._tool_spans[tool_call_id] = span + self._tool_call_parents.setdefault(tool_call_id, span.export()) + + def _end_tool_span(self, frame: Any) -> None: + tool_call_id = getattr(frame, "tool_call_id", None) + span = self._tool_spans.pop(tool_call_id, None) if isinstance(tool_call_id, str) else None + if span is None: + self._start_tool_span(frame) + span = self._tool_spans.pop(tool_call_id, None) if isinstance(tool_call_id, str) else None + if span is None: + return + span.log(output=getattr(frame, "result", None), metadata={"tool_call_id": tool_call_id}) + span.end() + + def _cancel_tool_span(self, frame: Any) -> None: + tool_call_id = getattr(frame, "tool_call_id", None) + span = self._tool_spans.pop(tool_call_id, None) if isinstance(tool_call_id, str) else None + if span is not None: + span.log(error=f"Tool call {tool_call_id} was cancelled") + span.end() + + def _log_transcription(self, frame: Any) -> None: + self._ensure_pipeline_span() + metadata = { + "user_id": getattr(frame, "user_id", None), + "timestamp": getattr(frame, "timestamp", None), + "language": _enum_value(getattr(frame, "language", None)), + "finalized": getattr(frame, "finalized", None), + } + span = start_span( + name="stt_transcription", + type=SpanTypeAttribute.TASK, + parent=self._pipeline_parent, + set_current=False, + ) + span.log( + output={"text": getattr(frame, "text", None), "result": getattr(frame, "result", None)}, + metadata={k: v for k, v in metadata.items() if v is not None}, + ) + span.end() + + def _start_tts_span(self, frame: Any) -> None: + self._ensure_pipeline_span() + context_id = getattr(frame, "context_id", None) or "__default__" + span = start_span( + name="tts_response", + type=SpanTypeAttribute.TASK, + parent=self._pipeline_parent, + set_current=False, + ) + metadata = { + "context_id": getattr(frame, "context_id", None), + "append_to_context": getattr(frame, "append_to_context", None), + } + span.log(metadata={k: v for k, v in metadata.items() if v is not None}) + self._tts_spans[context_id] = span + if context_id == "__default__": + self._tts_default_span = span + + def _append_tts_text(self, frame: Any) -> None: + span = self._tts_span_for_frame(frame) + if span is not None: + span.log(input={"text": getattr(frame, "text", None)}) + + def _log_tts_audio_metadata(self, frame: Any) -> None: + span = self._tts_span_for_frame(frame) + if span is None: + return + audio = getattr(frame, "audio", None) + metadata = { + "audio_size_bytes": len(audio) if isinstance(audio, (bytes, bytearray)) else None, + "sample_rate": getattr(frame, "sample_rate", None), + "num_channels": getattr(frame, "num_channels", None), + "num_frames": getattr(frame, "num_frames", None), + "context_id": getattr(frame, "context_id", None), + } + span.log(metadata={k: v for k, v in metadata.items() if v is not None}) + + def _end_tts_span(self, frame: Any) -> None: + context_id = getattr(frame, "context_id", None) or "__default__" + span = self._tts_spans.pop(context_id, None) + if span is not None: + span.end() + if context_id == "__default__": + self._tts_default_span = None + + def _tts_span_for_frame(self, frame: Any) -> Any | None: + context_id = getattr(frame, "context_id", None) or "__default__" + return self._tts_spans.get(context_id) or self._tts_default_span + + def _capture_metrics(self, frame: Any, processor: Any) -> None: + for metric in getattr(frame, "data", []) or []: + metric_type = type(metric).__name__ + if metric_type == "LLMUsageMetricsData": + self._llm_metrics.update(_llm_usage_metrics(getattr(metric, "value", None))) + self._llm_metadata.update(_metadata_from_metric(metric)) + elif metric_type == "TTFBMetricsData" and self._llm_span is not None: + value = getattr(metric, "value", None) + if isinstance(value, (int, float)) and not isinstance(value, bool): + self._llm_metrics["time_to_first_token"] = value + self._llm_metadata.update(_metadata_from_metric(metric)) + self._llm_metadata.update(_metadata_from_processor(processor)) + self._llm_metadata = _filter_llm_metadata(self._llm_metadata) + + def _log_error_frame(self, frame: Any) -> None: + self._ensure_pipeline_span(processor=getattr(frame, "processor", None)) + error = getattr(frame, "exception", None) or getattr(frame, "error", None) + if self._llm_span is not None: + self._llm_span.log(error=error) + self._llm_span.end() + self._llm_span = None + if self._pipeline_span is not None: + self._pipeline_span.log(error=error, metadata={"fatal": getattr(frame, "fatal", None)}) + + def _end_pipeline_span(self, frame: Any) -> None: + if self._pipeline_span is None: + return + metadata = {"terminal_frame": type(frame).__name__} + reason = getattr(frame, "reason", None) + if reason is not None: + metadata["reason"] = reason + self._close_child_spans() + self._pipeline_span.log(metadata=metadata) + self._pipeline_span.end() + self._pipeline_span = None + self._pipeline_parent = None + + def _close_child_spans(self) -> None: + if self._llm_span is not None: + self._end_llm_span() + for span in list(self._tool_spans.values()): + span.end() + self._tool_spans.clear() + for span in list(self._tts_spans.values()): + span.end() + self._tts_spans.clear() + self._tts_default_span = None + + def _close_all_open_spans(self) -> None: + self._close_child_spans() + if self._pipeline_span is not None: + self._pipeline_span.end() + self._pipeline_span = None + self._pipeline_parent = None + + +def _current_parent_export() -> str | None: + span = current_span() + if span == NOOP_SPAN: + return None + try: + return span.export() + except Exception: + return None + + +def _messages_from_context(context: Any) -> Any: + get_messages = getattr(context, "get_messages", None) + if callable(get_messages): + try: + return get_messages() + except Exception: + return getattr(context, "messages", None) + return getattr(context, "messages", None) + + +def _metadata_from_context(context: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + tools = getattr(context, "tools", None) + if tools is not None and not _is_not_given(tools): + converted = _tools_schema_to_openai_tools(tools) + if converted: + metadata["tools"] = converted + tool_choice = getattr(context, "tool_choice", None) + if tool_choice is not None and not _is_not_given(tool_choice): + metadata["tool_choice"] = tool_choice + return metadata + + +def _tools_schema_to_openai_tools(tools: Any) -> list[dict[str, Any]]: + standard_tools = getattr(tools, "standard_tools", None) + if standard_tools is None and isinstance(tools, list): + standard_tools = tools + ret: list[dict[str, Any]] = [] + for tool in standard_tools or []: + to_default_dict = getattr(tool, "to_default_dict", None) + if callable(to_default_dict): + payload = to_default_dict() + elif isinstance(tool, dict): + payload = tool + else: + continue + if isinstance(payload, dict) and payload.get("type") == "function": + ret.append(payload) + elif isinstance(payload, dict): + ret.append({"type": "function", "function": payload}) + custom_tools = getattr(tools, "custom_tools", None) + if isinstance(custom_tools, dict): + for values in custom_tools.values(): + if isinstance(values, list): + ret.extend(item for item in values if isinstance(item, dict)) + return ret + + +def _metadata_from_processor(processor: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + settings = getattr(processor, "_settings", None) + model = getattr(settings, "model", None) or getattr(processor, "model", None) + if isinstance(model, str): + metadata["model"] = model + provider = _provider_from_processor(processor) + if provider: + metadata["provider"] = provider + return metadata + + +def _metadata_from_metric(metric: Any) -> dict[str, Any]: + metadata: dict[str, Any] = {} + model = getattr(metric, "model", None) + if isinstance(model, str): + metadata["model"] = model + processor = getattr(metric, "processor", None) + if isinstance(processor, str): + provider = _provider_from_name(processor) + if provider: + metadata["provider"] = provider + return metadata + + +def _provider_from_processor(processor: Any) -> str | None: + module = getattr(type(processor), "__module__", "") + return _provider_from_name(module) + + +def _provider_from_name(name: str) -> str | None: + lowered = name.lower() + providers = { + "openai": "openai", + "anthropic": "anthropic", + "google": "google", + "gemini": "google", + "mistral": "mistral", + "cohere": "cohere", + "bedrock": "bedrock", + "aws": "bedrock", + "openrouter": "openrouter", + } + for needle, provider in providers.items(): + if needle in lowered: + return provider + return None + + +def _processor_name(processor: Any) -> str | None: + name = getattr(processor, "name", None) + if isinstance(name, str): + return name + if processor is not None: + return type(processor).__name__ + return None + + +def _is_pipeline_sink_processor(processor: Any) -> bool: + name = _processor_name(processor) + return isinstance(name, str) and name.endswith("::Sink") + + +def _filter_llm_metadata(metadata: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in metadata.items() if key in _ALLOWED_LLM_METADATA_FIELDS and value is not None} + + +def _llm_usage_metrics(usage: Any) -> dict[str, Any]: + metrics: dict[str, Any] = {} + prompt_tokens = getattr(usage, "prompt_tokens", None) + completion_tokens = getattr(usage, "completion_tokens", None) + total_tokens = getattr(usage, "total_tokens", None) + cache_read = getattr(usage, "cache_read_input_tokens", None) + cache_creation = getattr(usage, "cache_creation_input_tokens", None) + reasoning_tokens = getattr(usage, "reasoning_tokens", None) + for key, value in ( + ("prompt_tokens", prompt_tokens), + ("completion_tokens", completion_tokens), + ("tokens", total_tokens), + ("cache_read_input_tokens", cache_read), + ("cache_creation_input_tokens", cache_creation), + ("reasoning_tokens", reasoning_tokens), + ): + if isinstance(value, (int, float)) and not isinstance(value, bool): + metrics[key] = value + return metrics + + +def _tool_call_from_pipecat_call(call: Any) -> dict[str, Any] | None: + name = getattr(call, "function_name", None) + tool_call_id = getattr(call, "tool_call_id", None) + if not isinstance(name, str): + return None + arguments = getattr(call, "arguments", None) + return { + "id": tool_call_id, + "type": "function", + "function": { + "name": name, + "arguments": json.dumps(arguments if arguments is not None else {}, sort_keys=True), + }, + } + + +def _parse_jsonish(value: Any) -> Any: + if isinstance(value, str) and value and value[0] in '[{"': + try: + return json.loads(value) + except json.JSONDecodeError: + return value + return value + + +def _enum_value(value: Any) -> Any: + return getattr(value, "value", value) diff --git a/py/uv.lock b/py/uv.lock index b87faf43..a733575c 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -802,6 +802,12 @@ test-openai-http2 = [ { name = "pytest-asyncio" }, { name = "pytest-vcr" }, ] +test-pipecat = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-vcr" }, + { name = "websockets" }, +] test-pydantic-ai-logfire = [ { name = "logfire" }, { name = "pytest" }, @@ -995,6 +1001,12 @@ test-openai-http2 = [ { name = "pytest-asyncio", specifier = "==1.3.0" }, { name = "pytest-vcr", specifier = "==1.0.2" }, ] +test-pipecat = [ + { name = "pytest", specifier = "==9.1.1" }, + { name = "pytest-asyncio", specifier = "==1.3.0" }, + { name = "pytest-vcr", specifier = "==1.0.2" }, + { name = "websockets", specifier = "==15.0.1" }, +] test-pydantic-ai-logfire = [ { name = "logfire", specifier = "==4.32.1" }, { name = "pytest", specifier = "==9.1.1" }, @@ -2113,7 +2125,7 @@ dependencies = [ { name = "tzlocal" }, { name = "uvicorn" }, { name = "watchdog" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/cd/b17f7a7574870642be5af42bad831b3800e51529ed2dcdc10493a0a0db23/google_adk-2.3.0.tar.gz", hash = "sha256:4b95c99865a9a6b56e8abd3d3b816935f0db1cebbef328736a269b6537bd1df7", size = 3426370, upload-time = "2026-06-18T18:47:08.4Z" } wheels = [ @@ -2155,7 +2167,7 @@ dependencies = [ { name = "sniffio" }, { name = "tenacity" }, { name = "typing-extensions" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/fe/b796087493c3c55371aa58b9f264841ace5bfdf8c668cafa7afa33c44bec/google_genai-2.10.0.tar.gz", hash = "sha256:77912cd558cd7dfd5b75c25fd1c609e78d7954dde583331104022a46ea90f9ee", size = 600039, upload-time = "2026-06-24T01:33:18.157Z" } wheels = [ @@ -3193,7 +3205,7 @@ dependencies = [ { name = "sniffio", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, { name = "typing-extensions", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, { name = "uuid-utils", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, - { name = "websockets", version = "16.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, + { name = "websockets", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, { name = "xxhash", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, { name = "zstandard", marker = "extra == 'group-10-braintrust-test-langchain' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands')" }, ] @@ -4426,7 +4438,7 @@ wheels = [ [package.optional-dependencies] realtime = [ - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets" }, ] [[package]] @@ -4474,8 +4486,7 @@ dependencies = [ { name = "requests", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, { name = "types-requests", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, { name = "typing-extensions", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "websockets", version = "16.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-openai-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra != 'group-10-braintrust-test-pydantic-ai-logfire' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-livekit-agents')" }, + { name = "websockets", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/16/b79c1849125eb6d19cae98c21ff35caa2e55b5ec8d7a02b354b711917ef7/openai_agents-0.17.3.tar.gz", hash = "sha256:63b6dda6bd4fb51169e2a2cbd5d187a4e5ce823bbd15f965c8ed1d3b89072eec", size = 5406135, upload-time = "2026-05-19T01:28:15.971Z" } wheels = [ @@ -7645,13 +7656,6 @@ wheels = [ name = "websockets" version = "15.0.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version == '3.13.*'", - "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", -] sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/da/6462a9f510c0c49837bbc9345aca92d767a56c1fb2939e1579df1e1cdcf7/websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b", size = 175423, upload-time = "2025-03-05T20:01:35.363Z" }, @@ -7707,81 +7711,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] -[[package]] -name = "websockets" -version = "16.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version == '3.13.*'", - "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/74/221f58decd852f4b59cc3354cccaf87e8ef695fede361d03dc9a7396573b/websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a", size = 177343, upload-time = "2026-01-10T09:22:21.28Z" }, - { url = "https://files.pythonhosted.org/packages/19/0f/22ef6107ee52ab7f0b710d55d36f5a5d3ef19e8a205541a6d7ffa7994e5a/websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0", size = 175021, upload-time = "2026-01-10T09:22:22.696Z" }, - { url = "https://files.pythonhosted.org/packages/10/40/904a4cb30d9b61c0e278899bf36342e9b0208eb3c470324a9ecbaac2a30f/websockets-16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957", size = 175320, upload-time = "2026-01-10T09:22:23.94Z" }, - { url = "https://files.pythonhosted.org/packages/9d/2f/4b3ca7e106bc608744b1cdae041e005e446124bebb037b18799c2d356864/websockets-16.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72", size = 183815, upload-time = "2026-01-10T09:22:25.469Z" }, - { url = "https://files.pythonhosted.org/packages/86/26/d40eaa2a46d4302becec8d15b0fc5e45bdde05191e7628405a19cf491ccd/websockets-16.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde", size = 185054, upload-time = "2026-01-10T09:22:27.101Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ba/6500a0efc94f7373ee8fefa8c271acdfd4dca8bd49a90d4be7ccabfc397e/websockets-16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3", size = 184565, upload-time = "2026-01-10T09:22:28.293Z" }, - { url = "https://files.pythonhosted.org/packages/04/b4/96bf2cee7c8d8102389374a2616200574f5f01128d1082f44102140344cc/websockets-16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3", size = 183848, upload-time = "2026-01-10T09:22:30.394Z" }, - { url = "https://files.pythonhosted.org/packages/02/8e/81f40fb00fd125357814e8c3025738fc4ffc3da4b6b4a4472a82ba304b41/websockets-16.0-cp310-cp310-win32.whl", hash = "sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9", size = 178249, upload-time = "2026-01-10T09:22:32.083Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5f/7e40efe8df57db9b91c88a43690ac66f7b7aa73a11aa6a66b927e44f26fa/websockets-16.0-cp310-cp310-win_amd64.whl", hash = "sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35", size = 178685, upload-time = "2026-01-10T09:22:33.345Z" }, - { url = "https://files.pythonhosted.org/packages/f2/db/de907251b4ff46ae804ad0409809504153b3f30984daf82a1d84a9875830/websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8", size = 177340, upload-time = "2026-01-10T09:22:34.539Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fa/abe89019d8d8815c8781e90d697dec52523fb8ebe308bf11664e8de1877e/websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad", size = 175022, upload-time = "2026-01-10T09:22:36.332Z" }, - { url = "https://files.pythonhosted.org/packages/58/5d/88ea17ed1ded2079358b40d31d48abe90a73c9e5819dbcde1606e991e2ad/websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d", size = 175319, upload-time = "2026-01-10T09:22:37.602Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ae/0ee92b33087a33632f37a635e11e1d99d429d3d323329675a6022312aac2/websockets-16.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe", size = 184631, upload-time = "2026-01-10T09:22:38.789Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c5/27178df583b6c5b31b29f526ba2da5e2f864ecc79c99dae630a85d68c304/websockets-16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b", size = 185870, upload-time = "2026-01-10T09:22:39.893Z" }, - { url = "https://files.pythonhosted.org/packages/87/05/536652aa84ddc1c018dbb7e2c4cbcd0db884580bf8e95aece7593fde526f/websockets-16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5", size = 185361, upload-time = "2026-01-10T09:22:41.016Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e2/d5332c90da12b1e01f06fb1b85c50cfc489783076547415bf9f0a659ec19/websockets-16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64", size = 184615, upload-time = "2026-01-10T09:22:42.442Z" }, - { url = "https://files.pythonhosted.org/packages/77/fb/d3f9576691cae9253b51555f841bc6600bf0a983a461c79500ace5a5b364/websockets-16.0-cp311-cp311-win32.whl", hash = "sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6", size = 178246, upload-time = "2026-01-10T09:22:43.654Z" }, - { url = "https://files.pythonhosted.org/packages/54/67/eaff76b3dbaf18dcddabc3b8c1dba50b483761cccff67793897945b37408/websockets-16.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac", size = 178684, upload-time = "2026-01-10T09:22:44.941Z" }, - { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, - { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, - { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, - { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, - { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, - { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, - { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, - { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, - { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, - { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, - { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, - { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, - { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, - { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, - { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, - { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, - { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, - { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, - { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, - { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, - { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, - { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, - { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, - { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, - { url = "https://files.pythonhosted.org/packages/72/07/c98a68571dcf256e74f1f816b8cc5eae6eb2d3d5cfa44d37f801619d9166/websockets-16.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d", size = 174947, upload-time = "2026-01-10T09:23:36.166Z" }, - { url = "https://files.pythonhosted.org/packages/7e/52/93e166a81e0305b33fe416338be92ae863563fe7bce446b0f687b9df5aea/websockets-16.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03", size = 175260, upload-time = "2026-01-10T09:23:37.409Z" }, - { url = "https://files.pythonhosted.org/packages/56/0c/2dbf513bafd24889d33de2ff0368190a0e69f37bcfa19009ef819fe4d507/websockets-16.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da", size = 176071, upload-time = "2026-01-10T09:23:39.158Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8f/aea9c71cc92bf9b6cc0f7f70df8f0b420636b6c96ef4feee1e16f80f75dd/websockets-16.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c", size = 176968, upload-time = "2026-01-10T09:23:41.031Z" }, - { url = "https://files.pythonhosted.org/packages/9a/3f/f70e03f40ffc9a30d817eef7da1be72ee4956ba8d7255c399a01b135902a/websockets-16.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767", size = 178735, upload-time = "2026-01-10T09:23:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, -] - [[package]] name = "wrapt" version = "1.17.3" From 107ad5cc729ea786da002ef7c6adb4816f220892 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 7 Jul 2026 17:05:11 -0400 Subject: [PATCH 2/2] add attachments --- docs/pipecat-ai-integration-plan.md | 293 ------------------ .../integrations/livekit_agents/tracing.py | 14 +- .../integrations/pipecat/__init__.py | 3 +- .../integrations/pipecat/test_pipecat.py | 75 +++++ .../integrations/pipecat/tracing.py | 189 ++++++++++- py/src/braintrust/integrations/utils.py | 14 + 6 files changed, 268 insertions(+), 320 deletions(-) delete mode 100644 docs/pipecat-ai-integration-plan.md diff --git a/docs/pipecat-ai-integration-plan.md b/docs/pipecat-ai-integration-plan.md deleted file mode 100644 index d7648114..00000000 --- a/docs/pipecat-ai-integration-plan.md +++ /dev/null @@ -1,293 +0,0 @@ -# Pipecat AI Integration Research and Plan - -Research source: `https://github.com/pipecat-ai/pipecat`, cloned locally at `/tmp/pipecat`, commit `81f5be1e638188f9fbd3f386accc360e0db934d5` (main after `v1.4.0`). - -Braintrust instrumentation spec source: `https://github.com/braintrustdata/braintrust-spec/blob/main/docs/instrumentation-guide.md`, reviewed 2026-07-03. - -## Summary - -Pipecat is a real-time voice/multimodal agent framework. It is not a single model provider SDK; it orchestrates transports, STT, LLM, TTS, tool calls, metrics, workers, and pipelines through a frame-processing graph. - -The best Braintrust integration point is Pipecat's observer system, installed automatically by patching worker construction. This captures the framework-level semantics users care about without wrapping every provider-specific Pipecat service implementation. - -Recommended initial integration: - -- Package: `braintrust.integrations.pipecat` -- Import/distribution: `pipecat` / `pipecat-ai` -- Minimum supported Pipecat version: `1.3.0` -- Patch target: `pipecat.pipeline.worker.PipelineWorker.__init__` -- Behavior: inject a Braintrust `BaseObserver` into the worker's `observers` list; the observer creates Braintrust spans from Pipecat frames and metrics. - -## Relevant Pipecat APIs - -### Public orchestration APIs - -Current Pipecat applications are built around: - -- `pipecat.pipeline.pipeline.Pipeline` -- `pipecat.pipeline.worker.PipelineWorker` -- `pipecat.pipeline.worker.PipelineParams` -- `pipecat.workers.runner.WorkerRunner` -- `pipecat.observers.base_observer.BaseObserver` -- `pipecat.observers.base_observer.FramePushed` -- `pipecat.observers.base_observer.FrameProcessed` - -`PipelineWorker` accepts `observers: list[BaseObserver] | None` and forwards frame activity through a `WorkerObserver` proxy. This is the cleanest hook because it is public, non-invasive, and already intended for analytics/monitoring. - -### Frame events worth tracing - -Important frame classes in `src/pipecat/frames/frames.py`: - -- Pipeline lifecycle: `StartFrame`, `EndFrame`, `StopFrame`, `CancelFrame`, `ErrorFrame` -- Turn and voice activity: `UserStartedSpeakingFrame`, `UserStoppedSpeakingFrame`, `BotStartedSpeakingFrame`, `BotStoppedSpeakingFrame`, `InterruptionFrame` -- STT: `TranscriptionFrame`, `InterimTranscriptionFrame` -- LLM: `LLMContextFrame`, `LLMFullResponseStartFrame`, `LLMTextFrame`, `LLMFullResponseEndFrame` -- Tool calls: `FunctionCallsStartedFrame`, `FunctionCallInProgressFrame`, `FunctionCallResultFrame`, `FunctionCallCancelFrame` -- TTS/audio: `TTSStartedFrame`, `TTSTextFrame`, `TTSAudioRawFrame`, `TTSStoppedFrame` -- Metrics: `MetricsFrame` containing `TTFBMetricsData`, `TTFAMetricsData`, `ProcessingMetricsData`, `LLMUsageMetricsData`, `TTSUsageMetricsData`, `TextAggregationMetricsData`, `TurnMetricsData` - -### Existing Pipecat observability - -Pipecat already ships: - -- Observer API (`BaseObserver`) for non-intrusive frame monitoring. -- Logging observers (`LLMLogObserver`, `MetricsLogObserver`, `TranscriptionLogObserver`). -- Built-in OpenTelemetry tracing via `PipelineWorker(enable_tracing=True)` and `pipecat-ai[tracing]`. - -The Braintrust integration should use observers and native Braintrust logging as the primary implementation. Pipecat's OpenTelemetry support can be a future bridge, but it should not be required for the initial integration because it adds optional dependencies and mostly emits OTel attributes rather than Braintrust-shaped span `input`/`output`/`metadata`/`metrics`. If a future OTel bridge is added, it must use the spec's `braintrust.input_json`, `braintrust.output_json`, `braintrust.metadata`, `braintrust.metrics`, and `braintrust.span_attributes` attributes rather than ad-hoc OTel fields. - -Avoid excess normalization in integration code. Braintrust's span processing / serialization pipeline already normalizes objects before sending them to Braintrust, so the Pipecat integration should shape payloads into the required high-level schema but not defensively JSON serialize/deserialize, recursively convert every provider object, or stringify values just to make them serializable. - -## Minimum version recommendation - -Use `pipecat-ai>=1.3.0`. - -Reasoning: - -| Version | Finding | -| --- | --- | -| `0.0.108` | Last pre-1.x tag; supports Python `>=3.10`, has `PipelineTask`, observers, and tracing, but is pre-stable and has many APIs later deprecated/renamed. | -| `1.0.0` | First stable release; requires Python `>=3.11`; still centered on `PipelineTask` / `PipelineRunner`. | -| `1.2.0` | Still `PipelineTask` based; adds/changes resource APIs. | -| `1.3.0` | Introduces current `PipelineWorker` and `WorkerRunner`; `PipelineTask` / `PipelineRunner` become deprecated aliases. | -| `1.4.0` | Current stable tag in clone; keeps `PipelineWorker` observer API and adds more frame/metrics features. | - -Supporting `1.0.0` through `1.2.x` would require a parallel `PipelineTask.__init__` patcher for deprecated APIs. That adds complexity for APIs Pipecat itself moved away from. For a new Braintrust integration, start at the current worker architecture (`1.3.0+`). - -Implications for Braintrust test matrix: - -- Add `[tool.braintrust.matrix.pipecat-ai]` with `latest = "pipecat-ai==1.4.0"` and floor `"1.3.0" = "pipecat-ai==1.3.0"`. -- Add `pipecat-ai = "pipecat"` to `[tool.braintrust.vendor-packages]`. -- Skip the nox session on Python `<3.11` because Pipecat 1.x requires Python `>=3.11`. - -## Braintrust instrumentation spec alignment - -The Braintrust instrumentation guide constrains the Pipecat design in a few important ways: - -- Pipecat is an **agentic / realtime framework**, not a provider. The outer Pipecat run should be a `task` span. Individual model calls inside the run should be child `llm` spans, and tool executions should be child `tool` spans. -- Any `llm` span emitted by the Pipecat integration must use the completion API schema: - - `input`: ordered OpenAI-style message array, unless a provider-native normalizer is intentionally used. Pipecat's `LLMContext` already uses an OpenAI-like universal message format, so use `context.get_messages()` and materialize attachments. - - `output`: OpenAI Chat Completions-style choices array, e.g. `[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "..."}}]`. - - tool-call outputs: OpenAI-style `message.tool_calls`, with function `arguments` as a JSON-encoded string. - - `metadata`: must include `model` and `provider` when known, plus only allowed LLM request configuration fields, `tools`, `tool_choice`, and prompt provenance. -- Available tool definitions are **metadata**, not input messages. Convert Pipecat `ToolsSchema.standard_tools` / `FunctionSchema` / direct-function schemas to OpenAI-style `metadata.tools`. Do not log executable handlers or closures. -- Metrics must use only spec-approved keys. Do not emit Pipecat-specific metric keys such as `ttfb`, `ttfa`, `processing_time`, `turn_probability`, or `text_aggregation_time` in `metrics`. Map only the following when available: - - `tokens`, `prompt_tokens`, `completion_tokens` - - `time_to_first_token` for LLM streaming / first-token latency - - reasoning/cache/audio/image token keys if Pipecat exposes them with the same semantics - - `start` and `end` are handled by Braintrust span timing - Pipecat-specific timing/diagnostic values may be omitted or, for non-LLM framework spans only, placed in conservative metadata if they are useful and JSON-serializable. -- Inline media must be converted in-place to Braintrust attachments. For Pipecat this matters for `LLMContext.create_image_message()`, `LLMContext.create_audio_message()`, `input_audio` content parts, `TTSAudioRawFrame`, and any future generated media frames. - -## Proposed span shape - -Pipecat is an orchestration/framework integration. The integration should produce a spec-conformant agentic span tree while staying conservative about framework-only events. - -Recommended hierarchy: - -```text -pipecat_pipeline (task span, one per PipelineWorker run/session) - user_turn (task span, optional; one per turn) - stt_transcription (task span/event from TranscriptionFrame) - llm_response (llm span, one Pipecat model response) - (tool span for FunctionCallInProgress/Result) - tts_response (task span, text-to-speech operation) -``` - -LLM span requirements: - -- `span_attributes.type = "llm"` and name such as `"pipecat_llm_response"`. -- `input`: ordered OpenAI-style message array from the nearest `LLMContextFrame.context.get_messages()`. Preserve Pipecat's universal message dicts/lists directly when they already match the required shape; only materialize attachments and remove/skip clearly non-loggable runtime objects. Do not run broad recursive conversion or JSON round-trips. -- `output`: one OpenAI-style choice object: - - normal streamed text: `[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": aggregated_text}}]` - - model-requested tool calls: `finish_reason = "tool_calls"`, `message.content = null` or accompanying text, and `message.tool_calls` populated from `FunctionCallsStartedFrame` / `FunctionCallFromLLM`. -- `metadata`: `provider` and `model` when derivable from the Pipecat service/metrics frame; allowed request config fields only; `tools` and `tool_choice` from `LLMContext.tools` / `LLMContext.tool_choice` when present. Keep metadata values as normal Python dicts/lists/scalars; do not pre-stringify them for native Braintrust logging. -- `metrics`: only spec-approved metrics. Convert `LLMUsageMetricsData.value` to `tokens`, `prompt_tokens`, `completion_tokens`, plus cache/reasoning token keys when present. Convert LLM `TTFBMetricsData` to `time_to_first_token` only when it measures first streamed token/chunk latency. - -Tool span requirements: - -- `span_attributes.type = "tool"`. -- Span name should be the actual function/tool name, not a generic `function_tool`. -- `input`: arguments supplied by the model, as a dict/object when available. -- `output`: result from `FunctionCallResultFrame`. -- Correlate by `tool_call_id`. If a tool errors/cancels, log `error` and close the span. - -Task / media span requirements: - -- `pipecat_pipeline` and optional turn/STT/TTS spans use `span_attributes.type = "task"`. -- Pipeline `input` may include worker/pipeline metadata (`worker.name`, processor names, sample rates), but do not use an `llm` span for this framework metadata. -- `stt_transcription` output should be transcript text/JSON plus user/language metadata. -- `tts_response` input is text; output should either omit raw audio or contain generated audio as a Braintrust `Attachment` when audio capture is explicitly enabled. -- Do not log raw `TTSAudioRawFrame.audio` bytes inline. By default, record audio byte counts/sample rate/channel count as non-LLM task metadata only. If `capture_audio=True` is added, aggregate chunks into WAV and materialize an attachment in `output`, not metadata. - -### Attachment strategy - -Use attachments where the spec requires them, but keep capture conservative so the integration does not unexpectedly upload large real-time media streams. - -Initial implementation should: - -- Materialize inline media already present in logged LLM messages: - - `data:image/...;base64,...` in Pipecat `LLMContext` image content parts. - - `input_audio.data` from `LLMContext.create_audio_message()` and similar audio content parts. - - file/document data if Pipecat message content uses OpenAI-style `file.file_data`. -- Use the shared Braintrust attachment helpers (`_materialize_attachment`, `_materialize_chat_message_content_part`, `_normalize_chat_messages`) and pass `Attachment` objects in the logged `input`/`output`. Do not hand-build canonical `braintrust_attachment` dictionaries in the integration. -- Preserve remote URLs as URLs; do not fetch arbitrary remote media solely to create attachments. -- Preserve unsupported or failed-to-decode media values unchanged instead of dropping them. - -Initial implementation should not: - -- Attach every inbound `InputAudioRawFrame` / `AudioRawFrame`; continuous microphone streams are too large and noisy for default tracing. -- Attach `TTSAudioRawFrame` output by default. -- Add `attachments` lists in metadata or output. Attachments must live at the exact input/output leaf they replace. - -Follow-up / opt-in: - -- Add `capture_audio=True` only after the core integration is stable. When enabled, aggregate TTS output chunks per `TTSStartedFrame.context_id` / `TTSStoppedFrame.context_id`, encode a WAV, and place the resulting `Attachment` in `tts_response.output`. -- Consider a separate, bounded `capture_stt_audio=True` later for segmented STT tests or debugging, with size limits and no default continuous-stream capture. - -## Implementation plan - -### Package layout - -Create: - -- `py/src/braintrust/integrations/pipecat/__init__.py` -- `py/src/braintrust/integrations/pipecat/integration.py` -- `py/src/braintrust/integrations/pipecat/patchers.py` -- `py/src/braintrust/integrations/pipecat/tracing.py` -- `py/src/braintrust/integrations/pipecat/test_pipecat.py` - -Public API: - -```python -from braintrust.integrations.pipecat import setup_pipecat, wrap_pipeline_worker -``` - -### Patcher design - -Use one main patcher: - -- `PipelineWorkerInitPatcher` - - `target_module = "pipecat.pipeline.worker"` - - `target_path = "PipelineWorker.__init__"` - - wrapper mutates/extends the `observers` kwarg to include one `BraintrustPipecatObserver`, unless one is already present. - -Optional follow-up patchers: - -- `PipelineWorkerRunPatcher` only if observer lifecycle cannot reliably close root spans on all terminal paths. -- `PipelineTaskInitPatcher` only if we later decide to support Pipecat `1.0.0`-`1.2.x`. - -Manual wrapping: - -- `wrap_pipeline_worker(PipelineWorker)` should apply the worker init patcher to a provided class. -- A direct `BraintrustPipecatObserver` should also be exported for users who prefer explicit observer wiring without global patching. - -### Integration class - -```python -class PipecatIntegration(BaseIntegration): - name = "pipecat" - import_names = ("pipecat",) - distribution_names = ("pipecat-ai",) - min_version = "1.3.0" - patchers = (PipelineWorkerPatcher,) -``` - -### Auto instrumentation - -Implemented: - -- Export `PipecatIntegration` from `py/src/braintrust/integrations/__init__.py`. -- Add Pipecat to `braintrust.auto_instrument()` so users can call `braintrust.auto_instrument()` before constructing a `PipelineWorker`. -- Add `py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py` to verify import-order behavior in a subprocess. - -### Nox and dependency wiring - -Implemented: - -- `test-pipecat` dependency group with `websockets==15.0.1`; this is required because `pipecat-ai==1.3.0` imports `websockets` but does not install it transitively. -- `[tool.braintrust.matrix.pipecat-ai]` with `latest = "pipecat-ai==1.4.0"` and floor `"1.3.0" = "pipecat-ai==1.3.0"`. -- `[tool.braintrust.cassette-dirs]` entry for `pipecat = ["pipecat-ai"]` because the test suite is VCR-backed. -- `PIPECAT_VERSIONS = _get_matrix_versions("pipecat-ai")`. -- `test_pipecat(session, version)` nox session that skips on Python `<3.11`, skips Python `>=3.14` for Pipecat's current `onnxruntime` dependency, and runs `braintrust/integrations/pipecat/test_pipecat.py`. - -## Testing strategy - -### Primary tests: VCR-backed real Pipecat/OpenAI pipelines - -The implemented tests intentionally use VCR-backed real Pipecat/OpenAI pipelines rather than fabricated frames. This keeps the observer state machine tied to actual Pipecat service behavior and real provider payload shapes. - -Current coverage: - -1. `setup_pipecat()` injects a `BraintrustPipecatObserver` into newly constructed `PipelineWorker` objects. -2. Explicit `BraintrustPipecatObserver()` works without adding a duplicate injected observer. -3. Repeated `setup_pipecat()` and `wrap_pipeline_worker()` calls are idempotent. -4. A real `Pipeline([OpenAILLMService(...)])` receives an `LLMContextFrame`, calls OpenAI through Pipecat, and emits a `pipecat_pipeline` task span. -5. The real OpenAI/Pipecat response emits a spec-conformant `pipecat_llm_response` llm span with OpenAI-style message-array input, choices-array output, `metadata.provider`, `metadata.model`, and token metrics. -6. Metrics assertions verify only spec-approved metric keys are present, allowing Braintrust span timing keys (`start`, `end`) alongside token and first-token metrics. -7. Python `<3.11` nox sessions skip cleanly; Python `>=3.14` is skipped until Pipecat's `onnxruntime` dependency ships wheels. -8. `braintrust.auto_instrument()` is covered by a subprocess script that constructs and runs a real VCR-backed Pipecat/OpenAI pipeline. - -Cassette locations: - -- `py/src/braintrust/integrations/pipecat/cassettes/latest/` -- `py/src/braintrust/integrations/pipecat/cassettes/1.3.0/` - -Run locally: - -```bash -cd py -nox -s "test_pipecat(latest)" -nox -s "test_pipecat(1.3.0)" -OPENAI_API_KEY=sk-test-dummy-api-key-for-vcr-tests \ - nox -s "test_pipecat(latest)" -- -k test_setup_pipecat_traces_real_pipeline_frames -``` - -### Follow-up tests still worth adding - -The initial VCR suite does not yet cover every span shape described above. Useful follow-ups: - -- Real Pipecat tool-call behavior that creates child `tool` spans from actual `FunctionCallsStartedFrame` / `FunctionCallInProgressFrame` / `FunctionCallResultFrame` sequences. -- Real transcription and TTS service behavior, preferably with narrow VCR cassettes when provider traffic is involved. -- Multimodal `LLMContext` attachment materialization with real provider-compatible payloads. -- Terminal-frame variants (`StopFrame`, `CancelFrame`) and cancellation cleanup. - -## Additional refinements before implementation - -- **Split initial scope from follow-up scope.** Initial implementation should cover worker/session task spans, LLM response spans, tool spans, transcript spans, TTS text/audio metadata, and spec-approved token metrics. Defer runner/bus spans, OTel bridging, realtime-provider-specific event lifecycles, and opt-in audio attachments. -- **Make observer configuration explicit but small.** Consider `setup_pipecat(capture_audio=False, trace_turns=True)` and `BraintrustPipecatObserver(capture_audio=False, trace_turns=True)`. Avoid many knobs until real users need them. -- **Define state correlation carefully.** The observer will need per-worker/per-pipeline state for active pipeline span, active turn span, active LLM span, active TTS span by `context_id`, and active tools by `tool_call_id`. Keep this state on the observer instance, not module globals, so concurrent Pipecat workers do not cross-contaminate traces. -- **Prefer best-effort provider/model attribution.** Use Pipecat metrics data (`processor`, `model`) and processor class names to populate `metadata.model` / `metadata.provider` when clear. If provider cannot be determined, omit the child `llm` span or use a conservative provider value only if the spec allows it; do not invent provider names that would affect pricing semantics. -- **Preserve user span context.** If a Pipecat worker is created/run inside an existing Braintrust span, attach `pipecat_pipeline` beneath that current span. Child spans should attach through exported parent contexts rather than relying on global current-span state across observer queue tasks. -- **Do not make Braintrust logging affect the pipeline.** Observer code should never change frame flow, return values, or Pipecat error behavior. Conversion failures should preserve original values or omit optional fields, not raise into Pipecat observer processing. -- **Test minimal shaping.** Add tests that pass already-valid message dicts and assert they are preserved without stringification or JSON round-trips, while inline media leaves are materialized in place. - -## Risks and open questions - -- **Long-running sessions:** Voice agents can run indefinitely. The root span must close on all terminal frames and also during worker cancellation/cleanup. -- **Metric spec compliance:** Pipecat exposes useful metrics (`ttfb`, `ttfa`, processing time, turn prediction), but the Braintrust instrumentation guide only permits a fixed metric key set. The integration must map only semantically compatible values and omit or demote the rest. -- **Duplicate provider spans:** Pipecat framework LLM spans may overlap with direct provider integrations if users also instrument OpenAI/Anthropic/etc. Do not suppress provider leaf spans implicitly; document the possible duplication and keep Pipecat span structure spec-conformant. -- **Existing workers:** `setup_pipecat()` cannot inject into workers constructed before setup unless we add a `run()` patcher. Document setup-before-construction as the expected path. -- **Audio attachments:** Useful but potentially large. Keep raw audio capture off by default. -- **Multi-worker/bus support:** Initial scope should trace each `PipelineWorker`; later work can add runner/bus-level spans if users need cross-worker traces. -- **Pipecat 1.5.0+ changes:** Main already contains a deprecation note for `WorkerRunner(loop=...)` in 1.5.0, so keep patch targets focused on `PipelineWorker.__init__` and observer frames rather than runner internals. diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index 7adfab31..9f556efa 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -2,13 +2,12 @@ import asyncio import contextlib -import io import json import time -import wave from contextvars import ContextVar from typing import Any +from braintrust.integrations.utils import _pcm_to_wav from braintrust.logger import ( NOOP_SPAN, Attachment, @@ -692,17 +691,6 @@ def _pop_playback_audio(obj: Any) -> Attachment | None: ) -def _pcm_to_wav(audio: bytes, *, sample_rate: int, num_channels: int) -> bytes: - buffer = io.BytesIO() - with wave.open(buffer, "wb") as wav_file: - writer: Any = wav_file - writer.setnchannels(num_channels) # pylint: disable=no-member - writer.setsampwidth(2) # pylint: disable=no-member - writer.setframerate(sample_rate) # pylint: disable=no-member - writer.writeframes(audio) # pylint: disable=no-member - return buffer.getvalue() - - _AGENT_TURN_METADATA_FIELDS = ( "generation_id", "speech_id", diff --git a/py/src/braintrust/integrations/pipecat/__init__.py b/py/src/braintrust/integrations/pipecat/__init__.py index ed4e6bc7..6363e4a7 100644 --- a/py/src/braintrust/integrations/pipecat/__init__.py +++ b/py/src/braintrust/integrations/pipecat/__init__.py @@ -27,7 +27,8 @@ def setup_pipecat( The setup hook patches ``PipelineWorker.__init__`` so newly constructed workers receive a ``BraintrustPipecatObserver`` unless one was already - provided explicitly. + provided explicitly. Pass ``capture_audio=True`` to attach captured user + and TTS audio to the emitted Pipecat spans. """ set_default_observer_options(capture_audio=capture_audio, trace_turns=trace_turns) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 3230aa07..6da694bd 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -1,3 +1,5 @@ +# pylint: disable=protected-access,too-few-public-methods + import asyncio import importlib import inspect @@ -14,6 +16,7 @@ wrap_pipeline_worker, ) from braintrust.integrations.test_utils import verify_autoinstrument_script +from braintrust.logger import Attachment from braintrust.test_helpers import init_test_logger @@ -85,6 +88,69 @@ def _worker_runner_kwargs(**overrides): return kwargs +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_pipecat_observer_capture_audio_adds_tts_and_user_audio_attachments(memory_logger): + TTSStartedFrame = _import("pipecat.frames.frames.TTSStartedFrame") + TTSTextFrame = _import("pipecat.frames.frames.TTSTextFrame") + TTSAudioRawFrame = _import("pipecat.frames.frames.TTSAudioRawFrame") + TTSStoppedFrame = _import("pipecat.frames.frames.TTSStoppedFrame") + UserStartedSpeakingFrame = _import("pipecat.frames.frames.UserStartedSpeakingFrame") + UserAudioRawFrame = _import("pipecat.frames.frames.UserAudioRawFrame") + UserStoppedSpeakingFrame = _import("pipecat.frames.frames.UserStoppedSpeakingFrame") + + observer = BraintrustPipecatObserver(capture_audio=True) + first_chunk = b"\x00\x00\x01\x00" * 20 + second_chunk = b"\x02\x00\x03\x00" * 10 + + await observer.on_pipeline_started() + await observer._handle_frame(TTSStartedFrame(context_id="ctx")) + await observer._handle_frame(TTSTextFrame("hello from tts", aggregated_by="sentence", context_id="ctx")) + await observer._handle_frame(TTSAudioRawFrame(first_chunk, sample_rate=16000, num_channels=1, context_id="ctx")) + await observer._handle_frame(TTSAudioRawFrame(second_chunk, sample_rate=16000, num_channels=1, context_id="ctx")) + await observer._handle_frame(TTSStoppedFrame(context_id="ctx")) + await observer._handle_frame(UserStartedSpeakingFrame()) + await observer._handle_frame(UserAudioRawFrame(first_chunk, sample_rate=16000, num_channels=1, user_id="user-1")) + await observer._handle_frame(UserAudioRawFrame(second_chunk, sample_rate=16000, num_channels=1, user_id="user-1")) + await observer._handle_frame(UserStoppedSpeakingFrame()) + await observer.cleanup() + + logs = memory_logger.pop() + tts_span = _single_span(logs, "tts_response") + tts_audio = tts_span["output"]["audio"] + assert isinstance(tts_audio, Attachment) + assert tts_audio.reference["content_type"] == "audio/wav" + assert tts_audio.data.startswith(b"RIFF") + assert tts_span["output"]["audio_size_bytes"] == len(first_chunk) + len(second_chunk) + assert tts_span["output"]["sample_rate"] == 16000 + assert tts_span["output"]["num_channels"] == 1 + assert tts_span["output"]["num_frames"] == 60 + + user_span = _single_span(logs, "user_speaking") + user_audio = user_span["input"]["audio"] + assert isinstance(user_audio, Attachment) + assert user_audio.reference["content_type"] == "audio/wav" + assert user_audio.data.startswith(b"RIFF") + assert user_span["input"]["audio_size_bytes"] == len(first_chunk) + len(second_chunk) + assert user_span["input"]["num_frames"] == 60 + assert user_span["input"]["user_id"] == "user-1" + + observer_without_start = BraintrustPipecatObserver(capture_audio=True) + await observer_without_start.on_pipeline_started() + await observer_without_start._handle_frame( + UserAudioRawFrame(first_chunk, sample_rate=16000, num_channels=1, user_id="user-1") + ) + await observer_without_start._handle_frame( + UserAudioRawFrame(second_chunk, sample_rate=16000, num_channels=1, user_id="user-1") + ) + await observer_without_start._handle_frame(UserStoppedSpeakingFrame()) + await observer_without_start.cleanup() + + logs = memory_logger.pop() + auto_started_user_span = _single_span(logs, "user_speaking") + assert auto_started_user_span["input"]["num_frames"] == 60 + + @pytest.mark.vcr @pytest.mark.asyncio async def test_setup_pipecat_traces_real_pipeline_frames(memory_logger): @@ -163,8 +229,17 @@ def test_setup_and_wrap_pipeline_worker_are_idempotent(): assert PipecatIntegration.min_version == "1.3.0" assert setup_pipecat(project_name="test-project-pipecat-py-tracing") assert setup_pipecat(project_name="test-project-pipecat-py-tracing") + assert setup_pipecat(project_name="test-project-pipecat-py-tracing", capture_audio=True) assert wrap_pipeline_worker(PipelineWorker) is PipelineWorker + capturing_worker = _make_worker(Pipeline([IdentityFilter()])) + capturing_observers = getattr(getattr(capturing_worker, "_observer"), "_observers") + capturing_bt_observer = next( + observer for observer in capturing_observers if isinstance(observer, BraintrustPipecatObserver) + ) + assert capturing_bt_observer.capture_audio is True + + assert setup_pipecat(project_name="test-project-pipecat-py-tracing", capture_audio=False) explicit_observer = BraintrustPipecatObserver() worker = _make_worker(Pipeline([IdentityFilter()]), observers=[explicit_observer]) worker_observer = getattr(worker, "_observer") diff --git a/py/src/braintrust/integrations/pipecat/tracing.py b/py/src/braintrust/integrations/pipecat/tracing.py index 1771e904..e8ccee62 100644 --- a/py/src/braintrust/integrations/pipecat/tracing.py +++ b/py/src/braintrust/integrations/pipecat/tracing.py @@ -3,8 +3,8 @@ import json from typing import Any -from braintrust.integrations.utils import _is_not_given, _normalize_chat_messages -from braintrust.logger import NOOP_SPAN, SpanTypeAttribute, current_span, start_span +from braintrust.integrations.utils import _is_not_given, _normalize_chat_messages, _pcm_to_wav +from braintrust.logger import NOOP_SPAN, Attachment, SpanTypeAttribute, current_span, start_span try: @@ -69,6 +69,11 @@ def __init__(self, *, capture_audio: bool = False, trace_turns: bool = True, **k self._tool_spans: dict[str, Any] = {} self._tts_spans: dict[str, Any] = {} self._tts_default_span: Any | None = None + self._tts_audio: dict[str, bytearray] = {} + self._tts_audio_metadata: dict[str, dict[str, Any]] = {} + self._user_audio_span: Any | None = None + self._user_audio: bytearray | None = None + self._user_audio_metadata: dict[str, Any] = {} async def on_pipeline_started(self) -> None: self._ensure_pipeline_span() @@ -114,6 +119,12 @@ async def _handle_frame(self, frame: Any, *, processor: Any = None) -> None: self._end_tool_span(frame) elif frame_type == "FunctionCallCancelFrame": self._cancel_tool_span(frame) + elif frame_type in {"UserStartedSpeakingFrame", "VADUserStartedSpeakingFrame"}: + self._start_user_audio_span(frame) + elif frame_type in {"InputAudioRawFrame", "UserAudioRawFrame"}: + self._capture_user_audio(frame) + elif frame_type in {"UserStoppedSpeakingFrame", "VADUserStoppedSpeakingFrame"}: + self._end_user_audio_span(frame) elif frame_type == "TranscriptionFrame": self._log_transcription(frame) elif frame_type == "TTSStartedFrame": @@ -121,7 +132,7 @@ async def _handle_frame(self, frame: Any, *, processor: Any = None) -> None: elif frame_type == "TTSTextFrame": self._append_tts_text(frame) elif frame_type == "TTSAudioRawFrame": - self._log_tts_audio_metadata(frame) + self._log_tts_audio(frame) elif frame_type == "TTSStoppedFrame": self._end_tts_span(frame) elif frame_type == "MetricsFrame": @@ -307,25 +318,38 @@ def _append_tts_text(self, frame: Any) -> None: if span is not None: span.log(input={"text": getattr(frame, "text", None)}) - def _log_tts_audio_metadata(self, frame: Any) -> None: + def _log_tts_audio(self, frame: Any) -> None: span = self._tts_span_for_frame(frame) + if span is None and self.capture_audio: + self._start_tts_span(frame) + span = self._tts_span_for_frame(frame) if span is None: return - audio = getattr(frame, "audio", None) - metadata = { - "audio_size_bytes": len(audio) if isinstance(audio, (bytes, bytearray)) else None, - "sample_rate": getattr(frame, "sample_rate", None), - "num_channels": getattr(frame, "num_channels", None), - "num_frames": getattr(frame, "num_frames", None), - "context_id": getattr(frame, "context_id", None), - } + audio = _audio_bytes(frame) + metadata = _audio_frame_metadata(frame) + if audio is not None: + metadata["audio_size_bytes"] = len(audio) + if self.capture_audio: + context_id = getattr(frame, "context_id", None) or "__default__" + self._append_audio_chunk( + self._tts_audio, + self._tts_audio_metadata, + context_id, + audio, + metadata, + ) span.log(metadata={k: v for k, v in metadata.items() if v is not None}) def _end_tts_span(self, frame: Any) -> None: context_id = getattr(frame, "context_id", None) or "__default__" span = self._tts_spans.pop(context_id, None) if span is not None: + output = self._pop_tts_audio_output(context_id) + if output: + span.log(output=output) span.end() + else: + self._discard_tts_audio(context_id) if context_id == "__default__": self._tts_default_span = None @@ -333,6 +357,72 @@ def _tts_span_for_frame(self, frame: Any) -> Any | None: context_id = getattr(frame, "context_id", None) or "__default__" return self._tts_spans.get(context_id) or self._tts_default_span + def _start_user_audio_span(self, frame: Any | None = None) -> None: + if not self.capture_audio: + return + self._ensure_pipeline_span() + if self._user_audio_span is not None: + return + self._user_audio = bytearray() + self._user_audio_metadata = _audio_frame_metadata(frame) if frame is not None else {} + self._user_audio_span = start_span( + name="user_speaking", + type=SpanTypeAttribute.TASK, + parent=self._pipeline_parent, + set_current=False, + ) + + def _capture_user_audio(self, frame: Any) -> None: + if not self.capture_audio: + return + audio = _audio_bytes(frame) + if audio is None: + return + self._ensure_pipeline_span() + if self._user_audio_span is None: + self._start_user_audio_span() + if self._user_audio is None: + self._user_audio = bytearray() + self._user_audio.extend(audio) + _merge_audio_metadata(self._user_audio_metadata, _audio_frame_metadata(frame)) + + def _end_user_audio_span(self, frame: Any | None = None) -> None: + if self._user_audio_span is None: + return + if frame is not None: + _merge_audio_metadata(self._user_audio_metadata, _audio_frame_metadata(frame)) + output = _audio_output( + self._user_audio, + self._user_audio_metadata, + filename_prefix="user_speaking", + ) + if output: + self._user_audio_span.log(input=output) + self._user_audio_span.end() + self._user_audio_span = None + self._user_audio = None + self._user_audio_metadata = {} + + def _append_audio_chunk( + self, + audio_by_context: dict[str, bytearray], + metadata_by_context: dict[str, dict[str, Any]], + context_id: str, + audio: bytes, + metadata: dict[str, Any], + ) -> None: + audio_by_context.setdefault(context_id, bytearray()).extend(audio) + _merge_audio_metadata(metadata_by_context.setdefault(context_id, {}), metadata) + + def _pop_tts_audio_output(self, context_id: str) -> dict[str, Any]: + audio = self._tts_audio.pop(context_id, None) + metadata = self._tts_audio_metadata.pop(context_id, {}) + return _audio_output(audio, metadata, filename_prefix="tts_response") + + def _discard_tts_audio(self, context_id: str) -> None: + self._tts_audio.pop(context_id, None) + self._tts_audio_metadata.pop(context_id, None) + def _capture_metrics(self, frame: Any, processor: Any) -> None: for metric in getattr(frame, "data", []) or []: metric_type = type(metric).__name__ @@ -376,9 +466,15 @@ def _close_child_spans(self) -> None: for span in list(self._tool_spans.values()): span.end() self._tool_spans.clear() - for span in list(self._tts_spans.values()): + self._end_user_audio_span() + for context_id, span in list(self._tts_spans.items()): + output = self._pop_tts_audio_output(context_id) + if output: + span.log(output=output) span.end() self._tts_spans.clear() + self._tts_audio.clear() + self._tts_audio_metadata.clear() self._tts_default_span = None def _close_all_open_spans(self) -> None: @@ -389,6 +485,73 @@ def _close_all_open_spans(self) -> None: self._pipeline_parent = None +def _audio_bytes(frame: Any) -> bytes | None: + audio = getattr(frame, "audio", None) + if audio is None: + return None + try: + data = bytes(audio) + except Exception: + return None + return data if data else None + + +def _audio_frame_metadata(frame: Any) -> dict[str, Any]: + return { + "sample_rate": getattr(frame, "sample_rate", None), + "num_channels": getattr(frame, "num_channels", None), + "num_frames": getattr(frame, "num_frames", None), + "context_id": getattr(frame, "context_id", None), + "user_id": getattr(frame, "user_id", None), + "transport_source": getattr(frame, "transport_source", None), + "transport_destination": getattr(frame, "transport_destination", None), + } + + +def _merge_audio_metadata(target: dict[str, Any], metadata: dict[str, Any]) -> None: + existing_num_frames = target.get("num_frames") + chunk_num_frames = metadata.get("num_frames") + for key, value in metadata.items(): + if value is not None and key not in {"audio_size_bytes", "num_frames"}: + target[key] = value + if isinstance(chunk_num_frames, int) and not isinstance(chunk_num_frames, bool): + if isinstance(existing_num_frames, int) and not isinstance(existing_num_frames, bool): + target["num_frames"] = existing_num_frames + chunk_num_frames + else: + target["num_frames"] = chunk_num_frames + + +def _audio_output( + audio: bytearray | bytes | None, + metadata: dict[str, Any], + *, + filename_prefix: str, +) -> dict[str, Any]: + if not audio: + return {} + audio_bytes = bytes(audio) + sample_rate = metadata.get("sample_rate") + num_channels = metadata.get("num_channels") + suffix = f"_{sample_rate}hz_{num_channels}ch" if sample_rate and num_channels else "" + if isinstance(sample_rate, int) and isinstance(num_channels, int) and sample_rate > 0 and num_channels > 0: + attachment = Attachment( + data=_pcm_to_wav(audio_bytes, sample_rate=sample_rate, num_channels=num_channels), + filename=f"{filename_prefix}{suffix}.wav", + content_type="audio/wav", + ) + else: + attachment = Attachment( + data=audio_bytes, + filename=f"{filename_prefix}{suffix}.pcm", + content_type="audio/pcm", + ) + return { + **{k: v for k, v in metadata.items() if v is not None and k != "audio_size_bytes"}, + "audio": attachment, + "audio_size_bytes": len(audio_bytes), + } + + def _current_parent_export() -> str | None: span = current_span() if span == NOOP_SPAN: diff --git a/py/src/braintrust/integrations/utils.py b/py/src/braintrust/integrations/utils.py index d4257df6..cbde0089 100644 --- a/py/src/braintrust/integrations/utils.py +++ b/py/src/braintrust/integrations/utils.py @@ -11,11 +11,13 @@ import base64 import binascii +import io import mimetypes import os import re import time import warnings +import wave from collections.abc import Callable, Mapping from dataclasses import dataclass from numbers import Real @@ -446,6 +448,18 @@ def _infer_audio_mime_type(response: Any, response_format: Any = None) -> str: return "application/octet-stream" +def _pcm_to_wav(audio: bytes, *, sample_rate: int, num_channels: int) -> bytes: + """Wrap signed 16-bit PCM audio bytes in a WAV container.""" + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav_file: + writer: Any = wav_file + writer.setnchannels(num_channels) # pylint: disable=no-member + writer.setsampwidth(2) # pylint: disable=no-member + writer.setframerate(sample_rate) # pylint: disable=no-member + writer.writeframes(audio) # pylint: disable=no-member + return buffer.getvalue() + + def _extract_audio_output( response: Any, *,