From eefae1acc1efc848ab807c35c6377895d1d56e6b Mon Sep 17 00:00:00 2001 From: engkimo Date: Sun, 27 Sep 2026 15:41:40 +0900 Subject: [PATCH 1/2] feat: add framework-independent cascade diagnostics --- DESIGN.md | 14 +- MEMORY.md | 59 +- PROGRESS.md | 49 ++ README.md | 69 +- ROADMAP.md | 54 +- docs/cascade-analysis.md | 117 +++ docs/cascade-config-v2.md | 163 ++++ docs/cascade-design.md | 554 +++++++++++++ docs/cascade-issue-plan.md | 65 ++ docs/cascade-observation-api.md | 180 ++++ docs/demo-live-softphone.md | 5 + docs/diagnostic-agent-design.md | 32 +- docs/implementation-status.md | 274 +++++++ docs/library-integration.md | 16 + docs/provider-adapter-guide.md | 8 + examples/v2/configs/cascade.json | 90 ++ examples/v2/configs/realtime.json | 70 ++ examples/v2/manifests/aggregation.json | 14 + examples/v2/manifests/engine.json | 26 + examples/v2/manifests/input-resampler.json | 39 + examples/v2/manifests/llm.json | 29 + examples/v2/manifests/output-resampler.json | 39 + examples/v2/manifests/realtime.json | 48 ++ examples/v2/manifests/stt.json | 41 + examples/v2/manifests/tts.json | 36 + schemas/config.v2.schema.json | 769 ++++++++++++++++++ schemas/manifest.v2.schema.json | 437 ++++++++++ schemas/service-event.v1.schema.json | 258 ++++++ src/voxbench/control_plane/run_api.py | 501 +++++++++++- src/voxbench/engine_harness/harness.py | 2 + src/voxbench/engine_harness/plan.py | 3 +- src/voxbench/observability/__init__.py | 10 + .../observability/cascade_analysis.py | 641 +++++++++++++++ src/voxbench/observability/observer.py | 128 ++- src/voxbench/observability/service_events.py | 251 ++++++ src/voxbench/registry/config_views.py | 67 ++ src/voxbench/registry/service.py | 86 +- src/voxbench/registry/v2.py | 161 ++++ src/voxbench/schemas_export.py | 9 +- src/voxbench/schemas_v2.py | 264 ++++++ src/voxbench/synthetic_caller/offline.py | 2 + src/voxbench/verification/core.py | 2 + .../configs/v1-baseline.resolved.json | 1 + .../fixtures/configs/v1-overlay.resolved.json | 1 + tests/test_cascade_analysis.py | 548 +++++++++++++ tests/test_postgres_integration.py | 108 ++- tests/test_registry_v2.py | 500 ++++++++++++ tests/test_service_observations.py | 618 ++++++++++++++ 48 files changed, 7365 insertions(+), 93 deletions(-) create mode 100644 docs/cascade-analysis.md create mode 100644 docs/cascade-config-v2.md create mode 100644 docs/cascade-design.md create mode 100644 docs/cascade-issue-plan.md create mode 100644 docs/cascade-observation-api.md create mode 100644 docs/implementation-status.md create mode 100644 examples/v2/configs/cascade.json create mode 100644 examples/v2/configs/realtime.json create mode 100644 examples/v2/manifests/aggregation.json create mode 100644 examples/v2/manifests/engine.json create mode 100644 examples/v2/manifests/input-resampler.json create mode 100644 examples/v2/manifests/llm.json create mode 100644 examples/v2/manifests/output-resampler.json create mode 100644 examples/v2/manifests/realtime.json create mode 100644 examples/v2/manifests/stt.json create mode 100644 examples/v2/manifests/tts.json create mode 100644 schemas/config.v2.schema.json create mode 100644 schemas/manifest.v2.schema.json create mode 100644 schemas/service-event.v1.schema.json create mode 100644 src/voxbench/observability/cascade_analysis.py create mode 100644 src/voxbench/observability/service_events.py create mode 100644 src/voxbench/registry/config_views.py create mode 100644 src/voxbench/registry/v2.py create mode 100644 src/voxbench/schemas_v2.py create mode 100644 tests/fixtures/configs/v1-baseline.resolved.json create mode 100644 tests/fixtures/configs/v1-overlay.resolved.json create mode 100644 tests/test_cascade_analysis.py create mode 100644 tests/test_registry_v2.py create mode 100644 tests/test_service_observations.py diff --git a/DESIGN.md b/DESIGN.md index d26f418..a6a1a18 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -8,6 +8,12 @@ このドキュメントは Claude Code / Codex への実装ハンドオフ用。**§1（存在理由）と §2（非目標）を最優先で守ること。** 特定スタック（Gemini / Asterisk / 8kHz μ-law）の値は実装のどこにもハードコードしない。それらは `/examples` 配下のサンプルプラグイン設定として“データ”でのみ存在する。 +> 文書の位置付け（2026-09-18）：本書は設計契約で、実装済み機能の一覧ではない。初期v1の対象・構成図・技術候補と、現在動く構成は区別する。現在の実装・local candidate・残件は [implementation status](docs/implementation-status.md)、製品判断は [MEMORY.md](MEMORY.md) を参照する。 +> +> STT/LLM/TTSを独立モデルで組む拡張は [cascade設計案](docs/cascade-design.md) に定義する。`voxbench/v2`でrealtime/cascade、service role、入力/出力PCM chainを表し、既存v1の解決結果とhashを維持する提案である。cascadeは現在未実装。 +> +> 2026-09-18追加要件：Pipecat、他framework/middleware、middlewareを使わない直接接続の全てを対象にする。音声方式（realtime/cascade）と実行方式は別軸。図中のPipeCat wrapperは任意のruntime adapter例で、core・observer・診断の必須dependencyではない。 + --- ## 1. 存在理由：劣化モデル（この製品の背骨） @@ -30,17 +36,17 @@ AI音声エージェントは**ブラウザ（WebRTC）では動くのに、電 ## 2. スコープと非目標（ガードレール） ### 2.1 v1スコープ -- エンジン：Asterisk（chan_websocket）1種。 -- プロバイダ：Gemini Live 1種。 +- 初期設計対象：エンジンAsterisk（chan_websocket）、プロバイダGemini Live。 +- 現在のコミット済み通話経路はAsterisk AudioSocketで、Gemini LiveとOpenAI Realtimeのadapterがある。chan_websocketを含む初期目標全体の完了を意味しない。 - ただし**コードのどの経路もGemini/Asteriskを前提にしてはならない**。両者は必ずプラグイン＋manifest越しに扱う。コアは両者を知らない。 ### 2.2 非目標（やらないこと） -- **PipeCatを作り直さない。** オーケストレーションエンジンとして薄くラップするだけ。プロバイダ抽象（Gemini/OpenAI/Bedrock）はPipeCatに肩代わりさせる。 +- **PipeCatを作り直さない。** 通話アプリが選ぶframework/独自runtimeを維持し、VoxBenchは共通観測契約と薄い任意adapterを提供する。Pipecatや他middlewareの利用を強制せず、直接SDK/HTTP/WebSocket接続にも対応する。テスト通話の起動はframework非依存runtime/launcher契約へ分離する。 - **Langfuse / ClickHouse / Redis をコアに同梱しない。** Postgresファースト。制御プレーン自身がOTLPシンクになる。ClickHouse/Timescaleは `--profile scale` のopt-inのみ。 - **特定値をハードコードしない。** `8000`, `target_rms=3000`, `CLEAR_STREAM_AFTER_SECS`, `silence_duration_ms=600` 等は全て `/examples` のサンプルconfig/manifest内のデータ。コアやプラグイン実装本体に定数として書かない。 - **プロバイダ固有ロジックをコアに入れない。** turn_taking所有権・対応codec・許可override等は全てmanifestで宣言。 - **ブラウザストレージ（localStorage等）をUIで使わない。** -- v1で**マルチテナント認証基盤・課金・RBACは作らない**（単一組織セルフホスト前提。後付け可能な構造にはしておく）。 +- v1で**マルチテナント認証基盤・課金は作らない**（単一組織セルフホスト前提）。初期のRBAC非対象方針はproduction診断の [OIDC/operator認可設計](docs/diagnostic-agent-design.md#111-operator-authentication-の推奨) で更新された。独自password管理は作らず、production診断にはrole/authorization hookを設ける。認可は現在未実装。 ### 2.3 設計原則 - コアスキーマは「封筒」と「拡張点の契約」だけを持つ。具体パラメータは全てプラグインのJSON Schemaに逃がす。 diff --git a/MEMORY.md b/MEMORY.md index 9a657cc..b626b7f 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -1,6 +1,41 @@ # VoxBench project memory -このファイルは、今後の実装・設計セッションで維持すべきプロダクト判断を短く記録する。詳細設計は [診断エージェント設計・実装計画](docs/diagnostic-agent-design.md) を参照する。 +このファイルは、今後の実装・設計セッションで維持すべきプロダクト判断を短く記録する。 + +## 現在の状態を読む順序（2026-09-27） + +- 実装状況の入口は [implementation status](docs/implementation-status.md)。コミット済み、検証済みlocal candidate、計画、deployment検証残件を分ける。 +- 設計は `DESIGN.md`、[診断エージェント設計](docs/diagnostic-agent-design.md)、[cascade設計案](docs/cascade-design.md)を参照する。設計の存在だけで実装済みとは判定しない。 +- `PROGRESS.md`と日付付きSerena実装memoryは当時のsnapshot。古い「未実装」や進捗率を現在の状態として再利用しない。 +- 公開用worktreeのコミット基準は`ae575b6`。UI commandとlocal diagnostic SSEは未コミットのlocal candidateで、必要な新規module/test/Web fileも未追跡である。詳細と独立した安定化手順はstatus文書に記録した。 +- 2026-09-18にlocal candidateを含むPython 3.14.3のsuiteで330 passed / 4 skipped、Ruff、Web build成功を確認。Postgres実接続、browser、production認証、公開CIの検証とは区別する。 +- 同日C1実装後の最終検証：現在のPython suiteは408 passed / 4 skipped、Ruff成功。コミット済み基準へC1だけを加えたcheckoutは401 passed / 4 skipped、Ruff成功。78件のC1テストに加え既存テストも成功。後者はUI/SSE candidate不要で、dependencyは既存環境を共有した。 +- ユーザーは調査・計画・設計に続いて実装を指示した。candidateを暗黙の公開baselineにせず、cascade契約はコミット済みv1との互換性を基準に実装する。 + +## STT → LLM → TTS cascade（2026-09-18） + +- ユーザー要件：STT、LLM、TTSを個別モデル/providerで組んだcascadeに対応する。既存speech-to-speech realtimeも維持する。 +- 合意した方向：既存cascadeの観測を先に、テスト通話の実行を後に進め、実装をissueへ分割する。最初のprovider組合せは未選択。 +- issue化済み：[親issue #15](https://github.com/engkimo/voxbench/issues/15)と9件の作業issue（#16–#24）。担当範囲・依存関係・着手順は[issue計画](docs/cascade-issue-plan.md)を参照する。issue公開は機能実装の完了を意味しない。 +- 追加ユーザー要件：Pipecat限定にしない。Pipecat以外のframework/middlewareと、middlewareを使わずSDK/HTTP/WebSocketを直接接続する独自実装にも対応する。 +- Realtime/Cascadeという音声方式と、frameworkあり/なしという実行方式は独立した軸。coreの観測・診断契約に特定frameworkのframe/context/queueを必須化しない。接続言語もHTTP契約で限定しない。 +- 共通service観測契約とconformance fixtureを中心に置く。framework adapterは任意dependency、直接接続の例はframework不要で動かす。専用adapterのないアプリにもapplication-owned hook/HTTP経路を提供する。 +- テスト通話はframework非依存の`RuntimeAdapter`/application launcher契約越しに起動する設計。通話runtimeの管理主体、model service adapter、観測transportを分離する。 +- 現在もobserver/HTTP ingestとbuilt-in直接接続Realtime経路はPipecat非依存。Pipecat helperは薄い構築境界で、他framework専用adapter、cascade観測・実行、横断conformanceは未実装。 +- #17/C1の設定契約をlocal candidateとして実装した。`voxbench/v2`のrealtime/cascade union、service role/modality、input/output PCM chain、一意なcomponent ID、判断主体、manifest版/digest固定とoverlay検証に対応する。v1のresolved object/hashは維持する。現在の契約と使い方は[cascade config v2](docs/cascade-config-v2.md)。 +- turn/request/response/TTS segment/epochを明示相関する。並行処理のdurationを足してend-to-end latencyにしない。clockや相関の不足はindeterminateとする。 +- 文字列はアプリruntime内で必要だが、VoxBenchのservice観測はmetadata-only。transcript/prompt/LLM文/本文hashを標準保存・診断model送信しない。 +- 音声のmedia timeとrun timeをsegment単位で対応付ける。STTとTTS間にaudio duration/level preservationを適用しない。 +- C1設定契約だけでは実通話成功を意味しない。C2が観測専用runとservice event ingestを追加した後も、executionとv2音声録音は明示拒否する。受入条件とslice順序はcascade設計文書に置く。 +- #18/C2のmetadata-only service観測をlocal candidateとして実装した。v1/v2の観測専用runで、framework非依存の同一DTO/HTTP batchを使用する。collector/event、session/turn/request/parent/response/segment/epochを保持し、本文・URL・raw provider ID・secretのfieldを閉じる。契約は[cascade observation API](docs/cascade-observation-api.md)。 +- 同一event IDの同一payload再送だけを冪等とし、変更は409。未知parentをcoverage不足として残し、out-of-order到着後は再構築する。cancel request/ackとrun completeを別証拠とする。observer queueとrun event数をboundしdrop countを公開する。 +- #19/C3Aの決定論的因果解析をlocal candidateとして実装した。明示的なparent request chainだけを辿り、STT finalization、turn coordination、LLM dispatch/first output/first answer、aggregation、TTS dispatch/first PCM、local playback、end-to-end local writeを分離する。各値はevidence refs、scope、definition version、clock/uncertaintyを持つ。欠測は`unobserved`、未校正clock間と負の順序は`indeterminate`であり、近接timestampによる推測やservice durationの単純加算はしない。契約は[cascade analysis](docs/cascade-analysis.md)。 +- `llm.first_output`はanswer/reasoning/tool_call/otherの種別だけを保持し、本文を保持しない。tool-call continuationとretryは別operation。generation epoch/responseが異なる古いplaybackをend-to-endに再利用しない。 +- latency incidentはv2 configの明示的`service_latency_slos`があるobserved measurementだけに生成する。通常metadataからlatency SLO、STT精度、意味品質、発音品質、remote audible timeを推論しない。 +- C3A後もaudio segment mapping、inspector、provider/framework test project・adapter、test-call実行は未実装。v2 `/runs/observed`のみ対応し、`/runs`・`/runs/async`とv2 audio chunkは明示拒否する。 +- 2026-09-27のC2最終検証：現状は433 passed / 4 skipped、Ruff成功。`ef7deec`へC1+C2だけを加えUI/SSE candidateを除外したcheckoutは426 passed / 4 skipped、Ruff成功。25件のC2 focused test、standalone JSON Schema、SQLite repository再構築を確認。既存dependencyを共有し、実provider/framework/Postgres接続は未検証。 +- 2026-09-27のC3A最終検証：`ae575b6`基準の公開用worktreeは444 passed / 5 skipped、Ruff成功。`ef7deec`へC1–C3だけを加えUI/SSE candidateを除外したcheckoutは436 passed / 5 skipped、Ruff成功。10件のC3 focused testとSQLite restart再構築を確認。実Postgres restart testは追加したが環境変数未設定でskip、実provider/Pipecat/browserは未検証。 +- ユーザー指定の次ゴール：決定論的fake STT/LLM/TTS Cascadeを、application-owned direct実装（middlewareなし）とPipecat実装の2 test projectとして作る。同じscenario・観測契約・C3解析を通し、VoxBenchで検証、調整、比較評価できるようにする。Pipecatはoptional dependencyのままにし、実provider選定とは分ける。既存issueでは#22と#23のPipecat部分に対応する。 ## 診断エージェント（2026-08-07） @@ -32,7 +67,9 @@ VoxBenchに、選択中の通話を起点としてコード、ログ、設定、 - 既存のremote audio session cookieは録音取得専用で、製品全体のoperator authの代替にはしない。 - productionでの設定変更、実験実行、コード修正は、将来の承認付きactionとして診断ツールから分離する。 -### 推奨実装順序 +### Production診断への推奨実装順序 + +以下は残件の設計順序。local UI/SSE経路の存在は後述のcandidate状況とstatus文書で管理し、production永続化・orchestrationの完了とは扱わない。 1. Timeline/incident modelとprojectionを`run_api.py`から分離する。 2. Evidence resolverとcoverage summaryを実装する。 @@ -40,11 +77,25 @@ VoxBenchに、選択中の通話を起点としてコード、ログ、設定、 4. LLMなしのdeterministic diagnostic bundle APIを作る。 5. Diagnostic session/message/tool call/claimを永続化する。 6. Fake model、bounded orchestrator、claim/citation validatorを実装する。 -7. SSEとAsk VoxBench drawerを実装する。 -8. 型付きUI commandとguided investigationを実装する。 +7. 永続event storeをSSEと既存local Ask VoxBench candidateへ接続し、long-running job/cancel/reconnectを実装する。 +8. 型付きUI command candidateを安定化し、guided investigationを実装する。 9. OIDC operator authとauthorization hookを実装する。 10. Production model adapterとegress policyを接続する。 +### 型付きUI command・local agent candidate（2026-08-15実装、2026-09-18監査） + +この節は未コミットcandidateの機能記録。コミット済み製品の機能一覧ではない。SSEは作成済みeventのfinite replayで、質問による調査計画、継続配信、自動再接続、session容量/TTLは未実装。 + +- `POST /runs/{run_id}/ui-commands/resolve`を追加し、未信頼proposalをPydanticで型検証した後、run内のincident/evidence/artifact/recording/filter/time windowへ解決する。 +- Webに検証済みcommand dispatcherを追加した。run/incident選択、evidence focus、time window、panel表示、録音選択・再生・停止、比較、category/stage/direction filterをUI stateへ直接反映する。 +- command IDはclientでidempotentに扱う。適用、拒否、browser autoplay blockを結果として表示する。 +- 右railの`Agent UI command bridge`は、現在のrunからsafeな例を生成して、人間とfake agentが同じ契約を共同確認するための手動adapterである。 +- 任意URL、DOM selector、JavaScript、フォーム入力、設定保存、任意API呼び出しは引き続き許可しない。 +- process内memoryを使うlocal deterministic diagnostic sessionを追加した。最も強いincident、なければ最初のtyped eventを選び、引用と検証済みUI commandをordinal付きSSE eventで配送する。 +- Webの`Ask VoxBench`がSSE commandを自動実行し、結果をControl Planeへidempotentにacknowledgeする。これにより人間がJSONをpasteせず、fake agentがUIを直接動かすend-to-end経路を確認できる。 +- SSEは`Last-Event-ID`/`after`で再生可能。ack済みcommandへ異なる結果を送ると`409`で拒否する。 +- 未実装なのは永続session/event/result store、production model adapterとbounded orchestrator、long-running job/cancel、guided sequence、OIDC認可である。local deterministic adapterをproduction診断agentと表現しない。 + ### 環境接続時に決めること - 外部model providerとデータ処理地域。 diff --git a/PROGRESS.md b/PROGRESS.md index 5a1f4f6..1beb009 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,5 +1,54 @@ # 進捗 +> 本書は日付ごとの履歴。各節の未実装リスト・進捗率は当時のsnapshotであり、現在の状態は [implementation status](docs/implementation-status.md) を参照する。 + +## C3A: deterministic Cascade causal latency (2026-09-27) + +- #19をlocal candidateとして実装。保存済みmetadata-only service eventの明示的turn/request/parent/response/segment/epochだけからoperationを再構築する`cascade_analysis.py`を追加した。 +- STT finalization、turn coordination、LLM dispatch/first output/first answer、aggregation、TTS dispatch/first PCM、local playback、end-to-end local writeをendpoint evidence付きで分離した。multi-segment lifecycle durationを加算せず、最初のcausal playbackまでのcritical pathを構成する。 +- 欠測は`unobserved`、未校正clock domain間と負のendpoint順序は`indeterminate`。tool/reasoning outputはfirst spoken answerにせず、continuation/retryを別operationとしてparent chainを辿り、old epoch/response playbackを再利用しない。 +- run detail/recent/timeline/liveへparams・prompt・toolsを除外した`ai_mode`/`ai_components`を投影。明示的v2 latency SLOだけがwarning incidentを生成する。 +- 10件のfocused test、SQLite repository restart、opt-in実Postgres restart testを追加。`ae575b6`基準の公開用worktreeは444 passed / 5 skipped、Ruff成功。`git archive ef7deec`へC1–C3のみを加えUI/SSE candidateを除いたcheckoutも436 passed / 5 skipped、Ruff成功。5件の実Postgres testは接続未設定でskipした。 +- 次ゴールは、同じfake Cascade scenarioをmiddlewareなしdirect実装とPipecat実装の2 test projectで動かし、VoxBenchへ取り込んで検証・調整・比較評価すること。実provider選定とは分離し、Pipecatはoptional dependencyを維持する。 + +## C2: metadata-only cascade service observation (2026-09-27) + +- #18をlocal candidateとして実装。v1/v2観測runへ、特定framework/provider SDKに依存しない閉じた`service_events` DTOとPython observer helperを追加した。 +- STT/turn/LLM/aggregation/TTS/realtime/playback/cancel/failure境界、collector event namespace、session/turn/request/parent/response/segment/generation epochを正規化して既存timeline storeへ永続化する。 +- component/role/turn authority、safe scalar field、timezone/clock uncertainty、同一payload retry、conflict、欠落parent、batch/queue/run上限を検証。generic eventによるreserved kind/source/IDの迂回を拒否する。 +- transcript/prompt/generated text/content hash/raw payload/URL/secret/provider ID/cross-run fieldを契約に持たない。STT finalとturn commit、TTS first PCMとlocal playback write、cancel requestedとacknowledgedを別の証拠として残す。 +- v2は`/runs/observed`とmetadata-only eventのみ。execution/audio recording、causal latency、UIは後続issue。commit/push/公開は未実施。 +- 最終検証：現状433 passed / 4 skipped、Ruff成功。`git archive ef7deec`へC1+C2のみを加え、UI/SSE candidateを除いたcheckoutも426 passed / 4 skipped、Ruff成功。既存dependencyを共有したsource completeness検証である。 + +## C1: versioned realtime/cascade configuration (2026-09-18) + +- #17をlocal candidateとして実装。v1を変更せず、v2のRealtime/Cascade設定、役割・modality・音声形式、双方向PCM chain、一意ID、判断主体を検証する契約を追加した。 +- manifest版/digestとモデル選択を解決結果へ固定。同じmanifest identityの内容変更、cross-version/cross-mode overlay、不正params・IO・host要求・overrideを拒否する。 +- middleware不要のRealtime/Cascade設定例と版別JSON Schema、[契約ガイド](docs/cascade-config-v2.md)、78件の受入テストを追加。v1 base/overlayのcanonical JSON/hash、保存済みv1の読み戻しを検証した。 +- 最終検証：現状は408 passed / 4 skipped、Ruff成功。`git archive ef7deec`へC1だけを加えたcheckoutは401 passed / 4 skipped、Ruff成功。既存dependencyを共有し、独立したUI/SSE candidateや個人ファイルは後者に含めていない。 +- v2は設定解決のみ。run API・legacy音声処理で明示拒否し、観測・解析・録音mapping・UI・adapter・実通話の完了とは扱わない。commit/push/公開は未実施。 + +## Framework-independent cascade decisions and issues (2026-09-18) + +- Pipecat以外のframeworkとmiddlewareを使わない直接実装にも対応する要件を、`MEMORY.md`とSerenaの合意事項memoryに記録した。Realtime/Cascadeとframeworkあり/なしを独立した軸として設計へ反映した。 +- [親issue #15](https://github.com/engkimo/voxbench/issues/15)と9件の作業issue（#16–#24）を`engkimo`で公開した。各issueに実装範囲・受入条件・依存関係を記載し、[issue計画](docs/cascade-issue-plan.md)へ一覧を保存した。 +- 直接実装の観測例と共通conformance、任意framework adapter、framework非依存の実行launcherを別issueへ分けた。追加frameworkと最初のprovider組合せは未選択で、機能実装は未着手。 + +## Documentation consistency and cascade investigation (2026-09-18) + +- コミット基準`ef7deec`とUI command/local diagnostic SSE candidateを区別した実装inventoryを追加。必要な未追跡module/test/Web fileと安定化手順を明記した。 +- local candidateを含む現状を再検証：Python 3.14.3で330 passed / 4 skipped、Ruff、Web production build成功。Postgres実接続4件はskip、browser/実provider/公開CIは今回未検証。 +- `git archive ef7deec`にcandidateの12実装/testファイルだけを加えた一時checkoutでも同じchecksが成功。既存dependencyを共有したsource単位の検証で、個人ファイルや他の未追跡実装は含めていない。 +- 設計・README・memoryの現在状態への参照を揃え、古いtyped speech未実装記述と初期RBAC非対象方針を更新した。履歴の判断は現在判断へ書き換えない。 +- [cascade調査・設計案](docs/cascade-design.md)を追加。v2 config/manifest、独立STT/LLM/TTS、双方向PCM chain、typed service観測、明示相関、clock/録音mapping、キャンセル、UI、slice別受入条件を提案。 +- 今回は調査・計画・設計。cascade機能の実装、candidateのcommit/push/公開はしていない。 + +## Diagnostic UI/SSE local candidate (2026-08-12–15; audited 2026-09-18) + +- 型付きUI command resolver/dispatcher、local deterministic session、finite ordinal SSE replay、idempotent acknowledgementがlocal candidateとして存在する。 +- production model調査、質問に応じた調査計画、session/event永続化、容量/TTL、long-running job/cancel、client自動再接続、guided sequence、OIDC認可は未実装。 +- この節は未コミットcandidateの機能記録で、コミット済み機能の一覧とは分ける。 + ## Library UX: RTP packet tap health and clock contract (2026-07-25) - public `RtpPacketTapAdapter`と`RtpCaptureHealthSnapshot`を追加。既存appのRTP receive境界からtransient diff --git a/README.md b/README.md index 53648ef..eb18f1e 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,30 @@ -VoxBench is an early OSS implementation of the schema and registry foundation -described in `DESIGN.md`. +VoxBench is a pre-1.0 OSS implementation of AI voice-call observation, +verification, and a common-time-axis inspector. Architecture contracts are in +`DESIGN.md`; current implementation status is in +[the status inventory](docs/implementation-status.md). + +The inventory distinguishes committed features, verified local candidates, +planned features, and deployment validation. The typed agent UI-command and +local diagnostic SSE sections below describe an **uncommitted local candidate**, +not functionality available in the audited `ae575b6` checkout. Dated progress +and memory entries are historical snapshots. + +Independent STT → LLM → TTS configuration is implemented as a local candidate +for issue #17: [v2 contract and examples](docs/cascade-config-v2.md). +[Metadata-only service observation](docs/cascade-observation-api.md) is also a +local candidate. [Deterministic causal latency analysis](docs/cascade-analysis.md) +is implemented locally for issue #19. Recording maps, inspection, adapters and +execution remain planned; v2 execution APIs explicitly reject them. + +Pipecat is optional: the observation library/HTTP ingest and built-in direct +realtime bridge do not require it. The cascade target includes Pipecat, other +frameworks, and applications connecting STT/LLM/TTS services directly. Dedicated +framework adapters and cascade execution remain planned, not currently tested +compatibility with every middleware. The next goal is matched fake Cascade test +projects with and without Pipecat, using the same observation and analysis contract. Implemented so far: @@ -616,6 +638,42 @@ Open `http://127.0.0.1:5173/`. The Web UI can inspect a run timeline, compare tw runs, play stage recordings, watch live run status, and start an async run from an example payload. +### Test typed agent UI commands (local candidate) + +This section requires the complete local UI/SSE candidate file set listed in +[the status inventory](docs/implementation-status.md#local-uisse-candidate-audit). +It does not work from the audited committed baseline alone. + +After selecting a run, use **Ask VoxBench** in the right rail: + +1. Select **Investigate & guide UI**. +2. The local deterministic agent creates a diagnostic session, cites the + strongest typed incident or event, and delivers a validated `ui_command` over + SSE. +3. Confirm that the incident/evidence and shared cursor move without pasting or + executing JSON in the browser. +4. Confirm that the execution result changes to **Acknowledged by Control + Plane**. + +This adapter does not call an external model. It provides a deterministic +end-to-end path for diagnostic session creation, replayable SSE events, direct UI +guidance, and client acknowledgement. The submitted question does not change +the deterministic incident selection. SSE replays already-created events; +durable sessions, continuous investigation, automatic client reconnection, +session capacity/TTL, model orchestration, guided sequences and operator +authorization are pending. Open **Manual agent command bridge** to +exercise individual command types or rejection behavior. Select **Load current +example**, edit the scoped JSON, and then select **Validate & execute**. An +unknown or URL-like `evidence_ref` must be rejected without moving the UI. + +The browser never executes an untrusted proposal directly. It first posts the +proposal to `POST /runs/{run_id}/ui-commands/resolve`; the Control Plane resolves +run, incident, evidence, stage, artifact, and time targets from the persisted +timeline. Supported operations are `select_run`, `select_incident`, +`focus_evidence`, `set_time_window`, `open_panel`, `play_recording`, +`pause_recording`, `show_comparison`, and `apply_view_filter`. Arbitrary URLs, +selectors, JavaScript, form input, and API calls are not part of this contract. + For the shortest product walkthrough, use **Diagnose a call in 3 seconds** at the top of the page: @@ -654,9 +712,10 @@ steps are additionally persisted as safe correlated events: provider speech or interrupt notification, interrupt path, truncation position, playback queue clear, and completion. Selecting the derived incident shows that evidence chain on the shared cursor. Discarded queue duration is not presented as audible tail because -the remote listener's actual playout is not yet observed. Packet capture, -sample-accurate cross-clock alignment, and typed caller/assistant speech intervals -remain future work. +the remote listener's actual playout is not yet observed. Raw PCAP import +and sample-accurate cross-clock alignment remain future work. Provider-VAD caller +speech and local assistant playback intervals are implemented, as described below; +they do not establish remote audible playout. Directional RTP observations also produce transport evidence when loss reaches 1%, jitter reaches 30 ms, or MOS falls to 3.5 or below. Consecutive degraded diff --git a/ROADMAP.md b/ROADMAP.md index b04afc7..94b0ed2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,6 +3,10 @@ This roadmap describes direction, not delivery dates. Priorities may change as real call evidence and contributor feedback reveal better abstractions. +Current status is maintained in [the implementation inventory](docs/implementation-status.md). +Dated progress/memory entries are historical. Design proposals and local +candidates are not automatically available in the committed baseline. + ## Product north star Align every signal in an AI voice call on one timeline, then let an operator move @@ -24,7 +28,51 @@ system knows. - Postgres run persistence and a leased, fenced async job queue. - A three-second synthetic demo requiring no provider account or Asterisk. -See [PROGRESS.md](PROGRESS.md) for completed implementation slices. +See [PROGRESS.md](PROGRESS.md) for historical implementation slices and the +inventory for current availability and verification scope. + +## Now: stabilize the diagnostic UI candidate + +- Review and package typed UI-command resolution and local diagnostic SSE as a + complete, independent change, including currently untracked modules/tests. +- Verify an isolated intended checkout and the browser evidence/acknowledgement path. +- Keep the deterministic local adapter distinct from production model diagnosis. +- Define session limits, durable events/jobs, cancellation/reconnection, + guided investigation and OIDC authorization as separate acceptance work. + +## Next: separate STT, LLM and TTS models + +C1 configuration/capability contracts, C2 metadata-only service observation and +C3A deterministic causal latency analysis +are implemented as local candidates: [current v2 contract](docs/cascade-config-v2.md) +and [service observation API](docs/cascade-observation-api.md), plus the +[causal analysis contract](docs/cascade-analysis.md). Recording maps, inspector +support, adapters and execution remain planned. + +The next goal is a matched direct/Pipecat pair of deterministic fake Cascade +projects. Both will emit the common contract so VoxBench can validate, tune and +compare runtime behavior before real providers are selected. + +- Add versioned realtime/cascade configuration and role/modality capabilities + while preserving legacy config hashes. +- Observe existing cascades with metadata-only service events and explicit + turn/request/response/TTS-segment correlation. +- Separate STT finalization, turn coordination, LLM answer generation, text + aggregation, TTS first audio and local playback waits. +- Preserve clock uncertainty, cancellation epochs and segment recording maps. +- Add component model badges, service sublanes and comparable run evidence. +- Follow observation support with optional framework-independent test-call execution. +- Keep realtime/cascade topology independent of runtime choice. Support Pipecat, + other frameworks, and direct SDK/HTTP/WebSocket applications through the same + observation contract; add a shared conformance suite and framework-free example. +- Use optional framework adapters and a runtime/application-launcher contract + for test calls, with no mandatory Pipecat package. + +Observation-first and framework-independent support are product decisions; +contract details and the first provider/additional-framework combination remain +proposed or unselected. See [the cascade investigation and design](docs/cascade-design.md). +Published work items and dependencies are tracked in [the issue plan](docs/cascade-issue-plan.md) +and [parent issue #15](https://github.com/engkimo/voxbench/issues/15). ## Now: make real-call diagnosis obvious @@ -51,8 +99,8 @@ See [PROGRESS.md](PROGRESS.md) for completed implementation slices. ## Next: transport and packet proof -- Add pcap and live RTP packet-tap adapters with explicit capture-health - accounting. +- Add raw pcap import and wire the existing RTP packet-tap/capture-health adapter + to real deployment-specific receive paths and drop counters. - Preserve direction, clock-rate, extended sequence, arrival cadence, and capture-drop evidence without persisting packet payloads. - Add redacted SIP transaction and SDP-derived format metadata behind an diff --git a/docs/cascade-analysis.md b/docs/cascade-analysis.md new file mode 100644 index 0000000..b5b33ce --- /dev/null +++ b/docs/cascade-analysis.md @@ -0,0 +1,117 @@ +# Cascade causal latency analysis + +Implemented locally for [issue #19](https://github.com/engkimo/voxbench/issues/19). +The analyzer derives service waits only from explicitly related metadata-only +events. It does not inspect conversation text and does not require Pipecat or a +provider SDK. + +## Output + +`GET /runs/{run_id}/timeline` returns: + +- safe top-level `ai_mode` and `ai_components` projections; +- one `lanes.turns` analysis per observed LLM operation; +- observed latency intervals with source `cascade_causal_analysis_v1`; +- latency incidents only when the resolved v2 config contains an applicable SLO. + +An operation is scoped by turn, LLM request, response and generation epoch. +Segments and downstream request aliases remain separate. Retries and tool-call +continuations therefore remain distinct operations even when they belong to the +same turn. The analyzer walks explicit `parent_request_alias` links; it never +associates an event because its timestamp happens to be nearby. + +Every measurement includes: + +- `status`: `observed`, `unobserved` or `indeterminate`; +- start/end event references and the complete evidence-reference list; +- turn/request/response/segment/epoch scope; +- `definition` and `definition_version: cascade-latency/v1`; +- clock domain and alignment uncertainty when known; +- a stable reason alias when a value cannot be derived. + +## Measurements + +| Name | Endpoints | +| --- | --- | +| `stt_finalization_wait` | Independent `turn.speech_ended` → correlated `stt.final_emitted` | +| `turn_coordination_wait` | Correlated STT final → `turn.committed` | +| `llm_dispatch_wait` | Committed turn → actual LLM request start | +| `llm_first_output_wait` | LLM request start → first declared output of any kind | +| `llm_first_answer_text_wait` | LLM request start → first answer text eligible for speech | +| `text_aggregation_wait` | First answer text → each correlated segment ready for TTS | +| `tts_dispatch_queue_wait` | Segment ready → actual TTS request start | +| `tts_first_audio_wait` | TTS request start → first correlated PCM output | +| `output_playback_start_wait` | First TTS PCM → first matching local frame write | +| `end_to_end_local_response_wait` | Independent speech end → first causally matching local frame write | + +`llm.first_output` carries only an allowlisted `output_kind` value: `answer`, +`reasoning`, `tool_call` or `other`. It carries no output content. Reasoning or a +tool call does not satisfy the first-answer boundary. + +The end-to-end value stops at a local write. `remote_playout_observed` remains +false. It is not a remote audible-response measurement. + +## Critical path + +Each operation contains a `critical_path` projection for the first causally +matching playback segment. Its adjacent waits cover speech end, STT finalization, +turn coordination, LLM dispatch/answer, aggregation, TTS dispatch/first PCM and +local playback start. Later segments retain their own measurements. + +The analyzer does not add complete STT, LLM and TTS durations. Service work, +generation and playback can overlap; summing their lifecycle durations would +double-count time. A complete critical path instead reports the actual +end-to-end span, covered adjacent waits and any unexplained duration. + +## Missing and clock evidence + +- A missing endpoint returns `unobserved`, never zero. +- An unresolved or explicitly unobserved parent remains in coverage. +- Different clock domains return `indeterminate` until a calibrated mapping is + available. An uncertainty number alone does not invent that mapping. +- An end timestamp before its start returns `indeterminate`. +- A new generation epoch cannot reuse an older response's playback boundary. +- Missing terminal events keep `completion_observed` false; run completion does + not synthesize them. + +Observed same-domain durations sum the two endpoint alignment uncertainties when +both are known. If either endpoint uncertainty is unknown, the derived value +keeps uncertainty as null. + +## Explicit latency SLOs + +V2 configs can opt into a latency contract under `spec.observability`: + +```json +{ + "service_latency_slos": [ + { + "id": "local-response", + "measurement": "end_to_end_local_response_wait", + "max_ms": 800 + } + ] +} +``` + +IDs and measurement names are unique. `max_ms` must be finite and positive. +Only an observed measurement above its matching threshold creates a +`cascade_latency_slo_v1` incident. Without a contract, intervals report +`latency_threshold_status: not_configured` and ordinary service metadata creates +no latency incident. + +An SLO does not authorize transcript/audio retention and does not imply STT +accuracy, semantic answer quality, pronunciation quality or remote playout. + +## Persistence and current limits + +Analysis is reconstructed deterministically from persisted timeline events and +the pinned resolved config. No separate mutable analysis row is required. +SQLAlchemy/SQLite restart reconstruction is verified by default; an equivalent +real-Postgres restart test is opt-in through `VOXBENCH_TEST_POSTGRES_URL`. + +The next product goal is a pair of deterministic Cascade test projects: one +application-owned direct STT/LLM/TTS loop and one Pipecat-backed loop. Both will +emit the same common observations so VoxBench can compare, tune and evaluate the +runtime choices. That work follows this analyzer and remains separate from the +current C3 implementation. diff --git a/docs/cascade-config-v2.md b/docs/cascade-config-v2.md new file mode 100644 index 0000000..2dda580 --- /dev/null +++ b/docs/cascade-config-v2.md @@ -0,0 +1,163 @@ +# Versioned realtime and cascade configuration + +Implemented locally for [issue #17](https://github.com/engkimo/voxbench/issues/17). +This slice validates and resolves configuration. Metadata-only service observation +is now a separate local candidate described in +[cascade-observation-api.md](cascade-observation-api.md). V2 recording ingestion, +inspector support and test-call execution remain future slices. `/runs` and +`/runs/async` reject v2 before creating a run; `/runs/observed` accepts it for +typed service events. Legacy harness, synthetic generation and recording +verification reject it before writing audio. + +No Pipecat or other middleware SDK is required. The example plugins are static +capability declarations for deterministic validation, not live provider adapters. + +## Resolve an example + +Use the same registry API as v1: + +```python +from pathlib import Path +from voxbench.registry.service import RegistryService + +registry = RegistryService.from_files( + config_paths=["examples/v2/configs/cascade.json"], + manifest_paths=Path("examples/v2/manifests").glob("*.json"), +) +result = registry.resolve_config("example-cascade") +print(result.hash) +print(result.resolved["manifest_pins"]) +``` + +For realtime, use `examples/v2/configs/realtime.json` and `example-realtime`. +The CLI also accepts these documents: + +```bash +voxbench resolve-config \ + --config examples/v2/configs/cascade.json \ + --manifest examples/v2/manifests/engine.json \ + --manifest examples/v2/manifests/stt.json \ + --manifest examples/v2/manifests/llm.json \ + --manifest examples/v2/manifests/tts.json \ + --manifest examples/v2/manifests/aggregation.json \ + --manifest examples/v2/manifests/input-resampler.json \ + --manifest examples/v2/manifests/output-resampler.json +``` + +## Source contract + +Config and manifest documents both use `apiVersion: voxbench/v2`. Existing +unversioned manifests and `voxbench/v1` configs retain their original models, +default materialization and resolved hash behavior. + +- `spec.ai.mode` selects `realtime` or `cascade`, with closed discriminated shapes. + Realtime has `realtime`; cascade requires `stt`, `llm`, `text_aggregation`, `tts`. +- Every component has `id`, `plugin`, `manifest_version`, `params`. Services also + require a selected `model`. LLM/realtime may declare `system_prompt_ref` and + `tools`; STT/TTS may not. +- `spec.engine` has `kind`, `manifest_version`, `params`. +- `spec.media` has separate `input_pipeline` and `output_pipeline` lists. Each + stage has a globally unique `id`, `type`, plugin reference and existing host, + override and invariant settings. `type` is a category, not an identity. +- `input_audio` / `output_audio` select exact formats at service audio boundaries. + Audio formats require an encoding (`pcm16`, `pcm24`, `pcm32`, `float32`), a + positive integer `rate`, and positive integer `channels`. Text boundaries + forbid audio selections. + +Component IDs start with a lowercase letter and contain up to 64 lowercase +letters, digits, underscores or hyphens. `engine` and `application` are reserved. +IDs are unique across both PCM directions, services and text aggregation. + +## Manifest contract + +V2 retains plugin kinds `engine`, `provider`, `processor`, with explicit contracts: + +| Kind | Required contract | +| --- | --- | +| Engine | `engine_caps`: supported wire codecs, decoded input/output PCM, supported decision authorities | +| Provider | `service_caps`: roles, modalities, supported audio formats, lifecycle/cancellation capabilities and decision authorities | +| PCM processor | `io`: exact accepted/produced PCM and passthrough/rate-changing/format-changing behavior | +| Text aggregation processor | `text_io`: text input and output; audio invariants are forbidden | + +Provider roles have fixed modalities: STT audio→text, LLM text→text, TTS +text→audio, realtime audio→audio. Audio boundaries require nonempty supported +format lists; text boundaries forbid those lists. `supported_models`, when +nonempty, constrains the selected model. An empty model list leaves support +unverified; accepting a name does not establish live readiness. `model_selection` +states whether the model is explicit or deployment-pinned. + +Lifecycle flags declare partial output, final output, response boundaries and +usage support. Cancellation flags distinguish request, terminal acknowledgement, +transport abort and conversation truncation. They describe expected hooks and +capability, not observed provider behavior. + +Every engine/service/processor params object is checked against that manifest's +`param_schema`. Host requirements cannot be removed by stage settings. An audio +stage cannot replace pinned manifest IO or invent enforced invariants. Overrides +must use unique allowlisted targets with valid types. + +## Audio and decision boundaries + +The input chain connects engine decoded input to STT/realtime input. The output +chain connects TTS/realtime output to engine decoded output. Registry compares +complete formats at every adjacent boundary. Empty chains require equal endpoint +formats; a missing converter fails validation. STT and TTS audio are independent: +no preservation comparison is made through the text LLM. + +The wire codec is validated against the engine decode contract. Text LLM manifests +do not contain telephony codec capabilities. + +`spec.turn_taking` selects one `speech_detector`, one `end_of_turn` and optional +`interruption` authority. References identify `application`, `engine`, or a +configured component. The application-owned coordinator is an explicit source +selection; engine/components must declare the corresponding authority in their +caps. TTS cannot own any caller decision; a text LLM cannot own speech detection. +Multiple observations do not create additional authorities or commit a turn. + +V2 `spec.observability.service_latency_slos` can explicitly bind one positive +`max_ms` threshold to each named Cascade measurement. The list is empty by +default. Ordinary service metadata has no implicit latency threshold; see the +[causal analysis contract](cascade-analysis.md). + +## Resolution, pinning and overlays + +V2 lookup includes kind, name and manifest version. Re-registering an identical +manifest is idempotent; changing its effective contents under the same identity +fails. Publish a new manifest version instead. The registry stores its own copy +so mutating the registration result does not modify a pinned manifest. +V2 config registration also snapshots the supplied object; changes require +explicit re-registration. + +Resolved v2 objects materialize PCM manifest defaults and include sorted +`manifest_pins`, one per component and the engine. Each pin contains component +ID, kind, name, version and SHA-256 of the normalized manifest. Resolved hashes +include the selected models, aggregation policy/params and effective pins. +Config source schemas describe authored configs, not the additional resolution +metadata in `manifest_pins`. + +Overlays must explicitly retain `apiVersion: voxbench/v2` and the same AI mode. +Cross-version and cross-mode inheritance fail. Changing just STT, LLM or TTS +within cascade mode is supported; unrelated components retain their values. +Migration from v1 and mode replacement require a separate base today. + +`registry.config_views` supplies version-aware service/chain reads as independent +copies without adding v2 fields to stored v1 objects. These helpers do not +establish v2 run support. + +## Schemas and verification + +`python -m voxbench.schemas_export` generates `config.v2.schema.json` and +`manifest.v2.schema.json` alongside unchanged v1 documents. JSON Schema describes +the structural contract; registry validation also checks semantic relationships. + +`tests/test_registry_v2.py` covers valid modes, formats, roles, IDs, decisions, +params, overrides, manifest immutability/version selection and overlays. Frozen +v1 base/overlay fixtures assert canonical JSON/hash compatibility. A SQLAlchemy +repository roundtrip using SQLite checks stored v1 config preservation; it is +not a real Postgres connectivity test. A subprocess rejects middleware imports +while resolving both v2 modes. Run/legacy-audio tests verify explicit rejection +and no created run/audio artifacts. + +Credentials remain application/deployment inputs. References may be declared by +plugin params schemas; this slice introduces no credentials, content retention +or diagnostic-model egress. Safe projections for v2 runs belong to later slices. diff --git a/docs/cascade-design.md b/docs/cascade-design.md new file mode 100644 index 0000000..104d1a8 --- /dev/null +++ b/docs/cascade-design.md @@ -0,0 +1,554 @@ +# STT → LLM → TTS cascade support: investigation and design + +Status: design with C1, C2 and C3A implemented as local candidates, 2026-09-27. +The current C1 shape and examples are in [cascade-config-v2.md](cascade-config-v2.md). +The C2 service-event contract is in +[cascade-observation-api.md](cascade-observation-api.md). The +[causal analysis contract](cascade-analysis.md) is now implemented locally. Recording +maps, adapters and runtime remain planned. Current status is in +[implementation-status.md](implementation-status.md). + +## 1. Intended behavior and initial scope + +VoxBench should observe and diagnose both integrated speech-to-speech realtime +services and a cascade whose STT, text LLM and TTS use independent models/providers. +Operators should see the exact model combination, where response time was spent, +where audio first degraded, and what happened to in-flight work on interruption. + +The user agreed to observation-first planning, followed by optional test-call +execution, and requested issue-sized implementation slices. The explicit scope +includes Pipecat, other frameworks/middleware, and applications using provider +SDKs/HTTP/WebSockets directly without middleware. No particular provider or +additional framework is selected yet. Concrete contract shapes below remain +design proposals until implemented and verified. + +```text +caller audio -> input PCM processors -> STT -> turn/context coordinator + | + v + text LLM + | + v + text aggregation + | + v + TTS + | + v + output PCM processors -> local playback -> caller +``` + +Conversation orchestration remains owned by the existing application/runtime. +VoxBench supplies observation contracts, deterministic analysis, evidence +projections and thin optional runtime integrations. The diagnostic agent's +`ModelAdapter` remains separate from the conversation's LLM service. + +For the first release, the semantic graph has one STT, one LLM, one aggregator +and one TTS. It supports multiple turns, multiple TTS segments per response and +overlapping generation/playback. Arbitrary branching, multiple voices, provider +fallback, speculative execution by VoxBench and a workflow editor are later work. +An external application's speculative requests may be marked as such, but are +excluded from the committed-turn latency path. + +### 1.1 Independent compatibility dimensions + +| AI topology | Middleware/runtime | Required observation path | +| --- | --- | --- | +| Realtime | Pipecat or another framework | Optional framework hook/adapter → common observation contract | +| Realtime | Direct SDK/HTTP/WebSocket application | Application-owned hook → common observation contract | +| Cascade | Pipecat or another framework | Per-service/turn/playback hooks → common service observation contract | +| Cascade | Direct STT/LLM/TTS application | Per-service/turn/playback hooks → the same service observation contract | + +Telephony/WebRTC transport and implementation language are additional independent +choices. A Python SDK is a convenience; the versioned HTTP contract must also +support non-Python applications. This matrix is a target, not a claim that every +framework/provider already has a tested native adapter. + +Common observer, ingest, analyzers and UI must not require Pipecat classes or any +other framework's frame/context/queue types. A dedicated adapter maps available +boundaries; an application without one can use direct instrumentation. Unsupported +observations remain unobserved, and a framework's bot-start event is not promoted +to verified remote playout. Core must import successfully with no framework SDK. + +## 2. Findings from the current implementation + +| Existing boundary | Finding | Required change | +| --- | --- | --- | +| `schemas.py:AiConfig` / `VoiceSpec` | Strict v1 config requires one `provider`, `model`, `params` | Versioned realtime/cascade union | +| `schemas.py:ProviderCaps` | Audio-oriented rates, codec and turn-owner capabilities | Role and modality contracts per service | +| `registry/service.py:_validate_cross_fields` | Resolves one provider manifest and treats its caps as the call's AI contract | Resolve three services and validate each boundary | +| `registry/merge.py:deep_merge` | Recursively merges object fields, including incompatible shapes | Reject cross-mode/version inheritance unless explicitly replaced | +| `engine_harness/plan.py:build_stage_plan` | One ordered PCM pipeline, keyed by stage `type` | Independent input/output chains with stable stage IDs | +| `engine_harness/harness.py:run_once` | Produces nominal taps; does not run a real cascade | Keep execution readiness distinct from artifact generation | +| `realtime_providers/providers.py:RealtimeProviderSession` | PCM in/out and five lifecycle events; no STT/text/TTS handoff | Keep compatible; introduce separate runtime service adapters | +| `engine_harness/pipecat_adapter.py` | Only a thin `Pipeline([...])` construction boundary | Add a versioned optional observer/integration adapter | +| `observability/observer.py` | Generic scalar events, audio taps, bounded batch splitting | Typed service observations and service-aware recording mapping | +| `run_api.py:/v1/observations` | All referenced `stage` values must be in `media.pipeline` | Distinguish service components and PCM stages | +| `run_api.py:_create_running_run`, `StoredRun`, `models.py:Run` | One provider string is required throughout run projections/storage | Add AI mode/component projection; retain legacy provider field | +| `run_api.py` / `verification/core.py` | Audio adjacency and final-stage rules assume a single PCM chain | Validate each PCM chain separately; never compare STT audio to TTS audio for duration/level preservation | +| `web/src/App.tsx` / `types.ts` | One Provider lane and stage identity based on PCM stage names | STT/LLM/aggregation/TTS sublanes and component-aware filters | +| `control_plane/ui_commands.py` local candidate | Filter stages come from PCM lanes; evidence targets come from existing primitives | Include service component IDs after candidate stabilization | + +The current generic attribute sanitizer rejects selected sensitive string markers +and limits scalar sizes. It is not a field allowlist for transcript/prompt/text +content. Cascade privacy must therefore be enforced by a new closed typed +contract, not by assuming arbitrary strings are harmless aliases. + +## 3. Research and implications + +Pipecat documents separate STT, context, LLM, TTS and transport processors, and +text aggregation between streamed LLM output and TTS. This supports using a thin +adapter while the application/framework keeps orchestration ownership. +[Official pipeline documentation](https://docs.pipecat.ai/pipecat/learn/pipeline). + +Its metrics distinguish service TTFB, TTS first audio, LLM answer output, text +aggregation and user-to-bot latency. We should preserve a metric's measurement +boundary and translate seconds to milliseconds explicitly. A framework metric +without turn/request correlation stays service-level evidence; it cannot be +assigned to a particular turn by proximity alone. +[Official metrics documentation](https://docs.pipecat.ai/pipecat/fundamentals/metrics). + +The STT guide distinguishes speech-end-to-final-segment timing from turn-based +STT, where the server itself defines the turn boundary. We cannot assume all STT +providers expose an independent speech end or treat a configured P99 latency as +a measurement of the current call. +[Official STT latency documentation](https://docs.pipecat.ai/pipecat/fundamentals/stt-latency-tuning). + +These are design implications drawn from the documentation. The repository does +not pin or install Pipecat today, and the researched latest API is not a verified +local dependency. Choose and pin an adapter compatibility range during its +implementation, recording framework version and metric definitions. Do not copy +current API names into core assumptions. + +## 4. Versioned configuration proposal + +Keep `voxbench/v1` parsing, validation, resolution and serialized hashes intact. +Introduce `voxbench/v2` for the new shape. Dispatch by `apiVersion`, then +discriminate `spec.ai` by `mode`. Do not add defaults to a v1 resolved object or +rewrite previously stored runs. Generate version-specific JSON schemas alongside +the existing schema; add an explicit v1→v2 conversion command later. + +### 4.1 Semantic services and audio processors + +The v2 AI object has exactly one of these shapes: + +```yaml +# Semantic fragment of a proposed voxbench/v2 cascade config. +# Provider/model names are placeholders. This illustrative fragment omits +# required manifest_version and audio selections; use examples/v2 for full configs. +ai: + mode: cascade + stt: + id: stt-main + plugin: example-stt + model: pinned-stt-model + params: {} + llm: + id: llm-main + plugin: example-text-llm + model: pinned-llm-model + params: {} + system_prompt_ref: prompt-version-ref + tools: [] + text_aggregation: + id: text-aggregation-main + plugin: example-sentence-aggregator + params: {} + tts: + id: tts-main + plugin: example-tts + model: pinned-tts-model + params: {} +``` + +```yaml +# Alternate v2 AI object: integrated speech-to-speech. +ai: + mode: realtime + realtime: + id: realtime-main + plugin: example-realtime + model: pinned-realtime-model + params: {} + system_prompt_ref: prompt-version-ref + tools: [] +``` + +`spec.media` has `input_pipeline` and `output_pipeline`. Each PCM stage has a +run-unique `id`, plus its existing `type`, `plugin`, `params` and effective +manifest contract. IDs are globally unique across services, aggregation and PCM +processors. Using `type` as identity would collide when both directions contain +a resampler or AGC. + +Input processors connect decoded telephony PCM to STT/realtime input. Output +processors connect TTS/realtime output to decoded telephony PCM/framing. Empty +chains are allowed only when the boundary formats already match. STT, text LLM +and TTS are not placed in the PCM stage list. Aggregation is an explicit text +processor whose policy/version affects config hashes and comparison. + +Credential references are declared by each plugin's params schema; actual +credentials remain at deployment/runtime boundaries. Safe run projections use +declared aliases and allowlisted params. Prompt contents, tool inputs, voice +samples and private/custom voice identifiers do not become ordinary metadata. + +### 4.2 Capability manifests and validation + +Keep plugin kinds `engine`, `provider`, `processor`. STT/LLM/TTS remain provider +plugins with role-specific capabilities. Text aggregation is a processor with a +text contract, without audio invariants. Introduce a separately versioned v2 +manifest schema; existing unversioned manifests stay v1. + +Proposed provider `service_caps`: + +- `roles`: subset of `stt`, `llm`, `tts`, `realtime`. +- `input_modality` / `output_modality`: audio→text, text→text, text→audio, audio→audio. +- audio input/output formats: supported encodings, rates, channels per boundary. +- observed lifecycle support: partial/final output, response boundaries, usage. +- endpointing/turn-boundary behavior and supported ownership arrangements. +- cancellation behavior: request sent, terminal outcome acknowledgement, + transport abort support; conversation truncation separately if supported. +- declaration of model selection/readiness limitations and safe failure aliases. + +Registry hard failures cover missing/wrong-role manifests, invalid model params, +duplicate IDs, modality mismatch, adjacent PCM IO mismatch, missing resampler, +unsupported turn ownership, forbidden overrides and ambiguous authorities. + +Do not compare the telephony wire codec directly with a text LLM or PCM-only STT +provider's supported codecs. Validate the engine's codec/decode contract, PCM +processor formats, and service formats at their actual connection boundaries. +An application-owned integration may leave a capability unobserved, but that +produces readiness `unknown`, not a fabricated pass. + +Published resolution should pin manifest versions/digests as well as selected +models and aggregation policy. V2 overlays cannot silently switch AI mode or +inherit v1 shapes through `deep_merge`; require a new base or an explicit +conversion/replacement operation. STT-only or TTS-only changes within the same +mode remain ordinary overlays. + +### 4.3 Turn ownership + +Extend the v2 turn contract with separate `speech_detector`, `end_of_turn` and +`interruption` authority references. Each decision has one designated authority; +multiple observations may exist. A client VAD, STT endpointing notification and +semantic turn detector may cooperate, but STT `final` is not automatically a +committed conversational turn. Reject two independent coordinators that both +submit the same turn to the LLM. + +End-of-turn and barge-in need distinct settings/capabilities. TTS does not own +caller VAD. The initial optional runtime submits committed turns only; STT +revision and speculative LLM output stay outside that execution scope. + +## 5. Observation, correlation and privacy contract + +### 5.1 Identity + +Use `run_id` as the call anchor and safe aliases for the following relationships: + +| Identity | Scope and purpose | +| --- | --- | +| `component_id` | Stable configured STT/LLM/TTS/aggregation/PCM node | +| `turn_alias` | Committed conversational turn; may contain multiple STT segments | +| `request_alias` | One component operation/attempt; session alias for continuous STT | +| `parent_request_alias` | Explicit causal parent; absent when not known | +| `response_alias` | Assistant response, including separate LLM operations around tool execution | +| `segment_alias` | Text unit/TTS request and its audio, preserving order | +| `generation_epoch` | Local cancellation generation; separates old and new output | +| `collector_alias` | Distinguishes concurrent producers and their event ID namespaces | + +Do not persist provider item IDs, transcript-derived identifiers/hashes or raw +frame content. Streaming STT has a long-lived session and bounded segment +operations; do not pretend it has one request per audio chunk. A request retry +gets a new alias linked to the same operation/turn. Missing causal links remain +unknown rather than being inferred from nearest timestamps. + +### 5.2 Closed service events + +Add an optional `service_events` collection to the existing observation batch. +Use a discriminated, versioned `ServiceObservation` contract with closed fields +and enum-specific scalar payloads. Preserve existing generic batches unchanged. +Initial event families: + +- `stt.segment_started`, `stt.partial_observed`, `stt.final_observed`: + safe segment/revision aliases, final flag, counts if allowed; no text. +- `turn.speech_started`, `turn.speech_ended`, `turn.committed`: + authority, measured boundary, coverage; no invented waveform segmentation. +- `llm.request_started`, `llm.first_output`, `llm.first_answer_text`, + `llm.response_completed`: output kind, safe finish reason, usage if observed. +- `text.segment_ready`: parent LLM operation, segment ordinal and aggregation + contract. No text content or hash. +- `tts.request_started`, `tts.first_audio`, `tts.audio_progress`, + `tts.request_completed`: segment, actual format, generated media duration. +- `service.failed`, `service.cancel_requested`, `service.cancel_outcome`: + safe outcome alias; requesting cancellation is not observing its success. +- `playback.segment_enqueued`, `playback.segment_write_started`, + `playback.segment_write_stopped`, `playback.queue_cleared`, + `playback.stale_output_dropped`: response/segment/epoch and local media duration. + +The integration must distinguish service response headers, first answer text, +first PCM output and first local write. A TTS start frame is not sufficient proof +of first audio. Tool calls/reasoning output are not spoken answer text. Completion +observations do not imply all generated audio was written or heard. + +Typed events translate to existing `timeline_events` with reserved scalar keys, +`category=provider/conversation/buffer`, `stage=component_id` where appropriate, +and `correlation_alias=request_alias`. Parent/turn/segment aliases remain +allowlisted scalar fields. Validate component existence, role/event compatibility +and same-run relationships server-side. An absent parent may be reported as +missing while waiting for out-of-order batches; never attach a cross-run parent. + +The typed path enforces privacy regardless of producer behavior. Reserved service +event names through the generic path must pass the same validator, so generic +attrs cannot bypass it. Reject text/prompt/transcript/tool arguments, raw errors, +URLs, provider identifiers and secrets. User audio retention stays a separate +explicit permission. Cascade application's necessary external STT/LLM/TTS data +flow is separate from retention in VoxBench and from diagnostic-model egress. + +### 5.3 Storage and bounds + +Reuse the existing event table for the initial metadata contract; no unbounded +raw response table. Keep at most 16 scalar attrs in a normalized event and 128 +total timeline/service events per batch. Helpers split batches outside audio +callbacks. Emit lifecycle boundaries and bounded progress windows rather than +one event per token/sample. Define per-run event/queue budgets and visible +observation-drop counters; batch limits alone do not bound memory usage. + +Event IDs include a collector namespace and stable local ordinal. Retries reuse +the same ID and payload. Same-ID/same-payload is idempotent; a different payload +is a conflict. Lifecycle grouping must tolerate reordered batches, repeated +events and missing terminals, retaining `completion_observed=false`. + +Initially derive a safe `ai_mode` and `ai_components` projection from the pinned +resolved config. Keep existing run `provider` for compatibility: actual provider +for v1/realtime, reserved `cascade` for cascade. New consumers use components; +never concatenate credential/endpoint data into this label. Mode/components must +appear consistently in run detail, recent, timeline, live preview and compare. +Additional DB indexes/normalized component rows can follow measured needs. + +### 5.4 Framework-independent instrumentation and conformance + +Define service observation DTOs independently of all framework/provider SDKs. +Python helpers accept typed metadata and enqueue it without blocking media paths; +document the equivalent HTTP payloads for non-Python callers. Do not require a +Pipecat pipeline to start/register an observed run, correlate a turn or complete it. + +Provide one deterministic trace suite used by both direct integrations and optional +framework adapters. The adapter reports its version, runtime alias and actually +observed boundary coverage. Test the same multi-turn, segmented output, interruption, +missing-boundary, clock, retry and privacy scenarios through common projections. +Compatibility means the adapter passes these cases for its documented boundaries, +not that it synthesizes missing framework events. + +The no-middleware reference integration is a small application-owned STT/LLM/TTS +loop with fake service SDKs first. It owns coordination and emits only common +metadata; it can demonstrate real SDK calls after providers are selected. Optional +Pipecat and another framework adapter implement the same mapping. The second +framework is selected for validation later; no LiveKit or other adapter is claimed +to exist today. Optional dependency absence must never break the direct example. + +## 6. Timing and deterministic analysis + +Every derived latency includes start/end evidence refs, scope aliases, measurement +definition/version, clock domain and alignment uncertainty. Use local monotonic +timestamps plus a bounded run-clock mapping whenever available. Cross-domain +subtraction requires calibrated mapping; otherwise return `indeterminate`. + +| Measurement | Definition and limitation | +| --- | --- | +| STT finalization wait | Observed speech end → final segment used by the committed turn; undefined if independent speech end is unavailable | +| Turn coordination wait | Final required segment → committed turn; not a second STT inference measurement | +| LLM first-output wait | Actual operation start → first observed output, with output kind | +| LLM first-answer-text wait | Operation start → first answer text eligible for speech; not reasoning/tool-call latency | +| Text aggregation wait | First answer text in the source unit → segment ready for TTS; per-segment scope | +| TTS first-audio wait | Segment's actual TTS operation start → first PCM output; excludes pre-request queueing | +| TTS dispatch queue wait | Segment ready → TTS request actually started | +| Output/playback start wait | First corresponding TTS PCM output → first corresponding local frame write | +| End-to-end local response wait | Speech end → first local frame write for the response causally tied to that committed turn | +| Generated/written/discarded duration | Separate media-time counters by response/segment/epoch; none means remote heard duration | + +Do not add full STT, LLM and TTS operation durations as end-to-end response time. +Generation and playback overlap, and a response may contain several requests. +Compute actual causal intervals/critical path, preserving tool execution, +dispatch/aggregation waits and non-overlapping unexplained spans. If endpoints +cannot be linked, display available intervals without a complete decomposition. +For barge-in during existing playback, match the next committed response rather +than reusing an old response's playback timestamp. + +A framework-reported metric without endpoints stays an observed metric with its +native definition. Do not manufacture two timestamps from one duration. Never +label an observed text chunk as a token unless the adapter guarantees token +granularity. No observed request or audio boundary means `unobserved`, not zero. + +Initial diagnostics are evidence-based: + +- Service failure and safe outcome, with component scope. +- Policy/SLO overrun only with an explicit applicable latency contract. +- Interrupted generation and observed queue-discard/stale-output events. +- Local playback gap suspected only with the relevant continuity contract and + generation/playback evidence; active LLM text generation alone is insufficient. +- Existing audio duration/level/PCM-quality rules within each audio chain. + +STT recognition accuracy, semantic answer quality and pronunciation accuracy are +`not measured` without an opted-in reference/content evaluation. No WER, TTS +quality MOS or semantic score is inferred from ordinary metadata. Full-reference +audio scoring requires a valid same-content reference; caller PCM is not the +reference for a newly generated TTS reply. + +## 7. Audio taps and recording time mapping + +Useful tap boundaries include decoded caller input, STT input after input +processors, original TTS PCM output, each output processor and final local write. +Record only declared/allowed taps; ordinary service metadata does not opt in +conversation recording. Do not create WAVs for STT text or LLM output. + +Input and output chains have independent formats, directions and media origins. +Audio invariants compare adjacent PCM boundaries in the same chain/segment; +they do not assert duration or level preservation across speech recognition and +speech synthesis. + +Concatenated stage WAV time is not run wall time: there can be long pauses and +generated audio can precede paced writes. Introduce segment-level mapping records +with artifact/stage ID, response/segment aliases, recording sample start/count, +format, source clock/run-relative boundary and uncertainty. Source-generation +mapping and paced-write mapping are distinct. A source receive timestamp alone +does not prove when each generated sample was played. + +Timeline cursor → recording seek uses a known piecewise map. In gaps or when the +map is missing, show that boundary and offer media-time playback instead of +seeking by `run_ms` into a concatenated WAV. V1 playback remains compatible, with +its existing alignment limitations exposed. Verified UI commands must resolve +to the mapped recording position for v2. + +Recording storage currently has one artifact/buffer per run/stage and rewrites +the growing observed WAV. Start with compatible artifacts plus persisted mapping +metadata; high-throughput segmented streaming storage is separate hardening. + +## 8. Interruptions, cancellation and optional execution + +On an observed interruption, the application/runtime coordinator advances the +generation epoch, prevents old output from entering the playback queue, clears +queued frames, cancels unfinished LLM/TTS requests when supported, and continues +caller/STT input. Late callbacks/audio from old epochs are dropped with bounded +evidence. Provider cancellation acknowledgements and transport aborts are recorded +separately; unsupported cancellation does not prevent local stale-output suppression. + +The STT session is not canceled merely because assistant playback is interrupted. +TTS request completion is not response playback completion. Conversation-history +commit/truncation belongs to the application's policy; remote heard words cannot +be reconstructed from local writes alone. Realtime `truncate_audio` remains a +capability of the existing integrated provider, not a mandatory method on STT/TTS. + +For execution support, define separate `SttServiceAdapter`, `TextLlmServiceAdapter` +and `TtsServiceAdapter` runtime boundaries with readiness, typed outputs, safe +errors, cancellation semantics and close. Text necessarily exists inside the +application runtime but is not serialized into observation events. Optional +Pipecat factories construct framework processors; core does not recreate its +queues/context/turn machinery. Existing `RealtimeProviderSession` stays intact. + +Separate test orchestration from the chosen conversation implementation with a +framework-independent `RuntimeAdapter`/application-launcher boundary: readiness, +start a test call with pinned config/run ID/observation transport, cancel/stop, +and close. An application-owned direct runtime, a Pipecat runtime and another +framework runtime can implement it without changing core analysis. Capabilities +must state whether the runtime supports execution or observation only; inability +to launch a call does not prevent observing one. Do not wrap another framework +inside Pipecat or require its installation to run direct-provider tests. + +Execution requires exact model/format preflight, synthetic/fake adapters for +default tests, bounded queues/timeouts and resource shutdown. A nominal harness +must reject cascade execution as unsupported until a real execution adapter is +available, rather than reporting a successful conversation from generated taps. + +## 9. UI and diagnostic-agent integration + +- Show `Realtime` or `Cascade` and all configured model components near the run + selector, recent list and comparison. +- Keep the five timeline primitives and existing categories. Add STT, turn, + LLM, aggregation and TTS sublanes within provider/conversation/buffer views. +- Select a turn/response to view its explicit request/segment chain and latency + endpoints. Uncorrelated service metrics remain separate. +- Show generated, queued, written and discarded audio quantities separately. +- Display missing lifecycle/correlation/clock/playout coverage by component. +- Compare exact STT/LLM/TTS models, aggregation/turn policy, route, formats and + measurement definitions. Label multiple simultaneous changes explicitly. + Matched configuration still does not imply deterministic provider output. +- Extend component filters/evidence resolution in the typed UI-command candidate + after stabilization. Reuse validated primitives instead of arbitrary selectors. +- Deterministic diagnostic bundles include component coverage and causal metrics. + A future diagnostic model receives safe metadata; cascade support does not + automatically permit transcript/audio egress. + +## 10. Implementation sequence and acceptance gates + +Each slice must leave v1 realtime/observer workflows working. These cascade slices +are distinct from the diagnostic-agent Phase 0–5 roadmap. + +Published tasks: [issue plan and dependencies](cascade-issue-plan.md), under +[parent issue #15](https://github.com/engkimo/voxbench/issues/15). Publication tracks +planned work; it does not change the implementation status of these proposals. + +| Slice | Work | Acceptance gate | +| --- | --- | --- | +| S0: baseline stabilization | Review/package the existing UI/SSE candidate independently; current inventory and document policy | Isolated intended file set passes checks; UI verified; feature commit/status explicit | +| C1: versioned contracts | v2 config/manifest unions, stable IDs, input/output PCM chains, central normalized read helpers | Legacy resolved bytes/hashes unchanged; invalid roles, formats, overlays and ownership fail | +| C2: service observation | Typed helper/batch path, event normalization, aliases, limits, metadata-only fixtures | Fake multi-turn/segmented run ingests without content; retries/conflicts/reorder and missing boundaries handled | +| C3a: causal analysis | Component projection, causal grouping, deterministic latency | Overlap/clock/cancel fixtures yield correct values or indeterminate; Postgres reconstruction matches | +| C3b: audio chains and recording maps | Per-chain verification and persisted segment tap mappings | Input/output identities and formats remain independent; cursor seek preserves gaps and uncertainty | +| C4: cascade inspector | Component model badges, sublanes, latency/evidence view, comparison and mapped listening | Operator identifies STT wait vs aggregation vs TTS/playback; unknowns visible; v1 playback preserved | +| C5a: direct integration and conformance | No-middleware example, common trace suite and language-independent HTTP guide | Framework SDKs absent; same semantic fixtures/coverage rules pass; no callback blocking/content retention | +| C5b: optional framework integrations | Versioned Pipecat adapter and one other framework adapter | Same conformance suite; only real available boundaries mapped; framework-specific packages optional | +| C6: optional test-call execution | Framework-independent runtime/launcher contract and service factories, fake first, then selected combination | Direct runtime works without Pipecat; readiness, overlap, interruption stale-drop and complete cleanup verified | + +C1–C3 can target the committed observation contracts without depending on the +local Ask VoxBench implementation. S0 is required before making that candidate +a shipped dependency. Record each implemented slice/commit in the status inventory. + +The proposed first cascade implementation PR is C1 only: closed schemas, +registry validation, v1 regression fixtures, examples and updated generated +schemas. C2 then proves metadata-only cascade observation before a live provider +or Web feature is added. + +After C3A, the user-designated next goal is a matched pair of deterministic fake +Cascade test applications: one direct application-owned STT/LLM/TTS loop and one +Pipecat-backed loop. Both use the same events, scenarios and analysis outputs so +VoxBench can compare runtime behavior without making Pipecat a core dependency. +This combines the proof work planned in C5A/C5B; real providers remain a later, +explicitly selected validation step. + +### Required fixtures + +1. One committed turn and two TTS segments; LLM completes after playback starts. +2. Multiple STT partial/final revisions; one final is selected by the coordinator. +3. Two overlapping turns and an interruption: late old-epoch TTS output is dropped. +4. Missing STT speech-end or request linkage: no invented finalization/e2e latency. +5. Separate uncalibrated clocks: cross-boundary timing is indeterminate. +6. Failed/unsupported cancellation: local stale-output suppression still holds. +7. Tool-call-only first LLM output, then an answer from another operation. +8. Generic reserved-name bypass, text/secret/URL fields and cross-run refs rejected. +9. Duplicate event delivery and changed-payload collision; collector namespaces. +10. Duplicate processor types in input/output chains and differing rates. +11. Same v1 config before/after schema changes: exact resolved object/hash stable. +12. Gapped concatenated recording and early TTS generation: cursor seeks using the + correct segment/write map, or explicitly cannot align. +13. Postgres restart reconstructs components/events/maps/derived evidence. +14. Metadata-only run with no WAV/transcript: accuracy/quality remains not measured. +15. No Pipecat/other framework SDK installed: core and direct cascade example work. +16. Direct, Pipecat and second-framework traces normalize to the same evidence + semantics for equivalent observed boundaries; missing boundaries stay unknown. +17. Observation-only application cannot be launched: readiness says unsupported + execution while observation remains available. + +## 11. Decisions still needed for environment integration + +- Existing cascade observation precedes optional execution; identify the first + actual application whose observed boundaries will validate the design. +- First STT/LLM/TTS combination, exact model versions, language and deployment + regions; credentials are not needed for C1–C4. +- Existing runtime (direct/Pipecat/other), its version and actual available hooks; + choose a second framework to demonstrate core-independent integration. +- Turn detection, aggregation, interruption policy and applicable latency SLOs. +- Which audio taps/content evaluation are explicitly permitted, with retention + and recording playback authorization. + +The versioned config, causal observation and compatibility design can proceed +without selecting vendors. These environment choices become prerequisites for +an actual integration/execution validation, not permission to fake its evidence. diff --git a/docs/cascade-issue-plan.md b/docs/cascade-issue-plan.md new file mode 100644 index 0000000..aacd9dd --- /dev/null +++ b/docs/cascade-issue-plan.md @@ -0,0 +1,65 @@ +# Cascade and framework-independent issue plan + +Recorded 2026-09-18. GitHub issues were created using the verified `engkimo` +connector account. Issue publication is complete; feature implementation and +local source/document commit publication are not. + +Parent: [#15: Support framework-independent realtime and STT → LLM → TTS diagnostics](https://github.com/engkimo/voxbench/issues/15) + +## Agreed direction + +Support both integrated realtime and independent STT → LLM → TTS, using Pipecat, +other frameworks/middleware, or direct SDK/HTTP/WebSocket application code. +Observation support comes first, optional test-call execution follows. Core +contracts and analysis must not require framework SDKs or frame/context types. + +Current boundaries and verification: [implementation-status.md](implementation-status.md). +Contract details and fixtures: [cascade-design.md](cascade-design.md). +The detailed design is currently local; public issue bodies are self-contained. + +## Issue inventory and prerequisites + +| Slice | Issue | Scope | Prerequisites | +| --- | --- | --- | --- | +| S0 | [#16](https://github.com/engkimo/voxbench/issues/16) | Stabilize the typed diagnostic UI/SSE candidate as a complete change | Independent | +| C1 | [#17](https://github.com/engkimo/voxbench/issues/17) | Add versioned realtime/cascade configs and role-specific service manifests | Independent | +| C2 | [#18](https://github.com/engkimo/voxbench/issues/18) | Add metadata-only cascade service observations with causal identities | #17 | +| C3A | [#19](https://github.com/engkimo/voxbench/issues/19) | Derive cascade latency from explicitly correlated service boundaries | #18 | +| C3B | [#20](https://github.com/engkimo/voxbench/issues/20) | Map cascade recording segments to run time and verify audio chains independently | #17, #18 | +| C4 | [#21](https://github.com/engkimo/voxbench/issues/21) | Show cascade components, causal waits and evidence coverage in the call inspector | #19, #20 | +| C5A | [#22](https://github.com/engkimo/voxbench/issues/22) | Add a no-middleware cascade example and shared observation conformance suite | #18 | +| C5B | [#23](https://github.com/engkimo/voxbench/issues/23) | Add optional framework adapters that share the direct cascade observation contract | #22 | +| C6 | [#24](https://github.com/engkimo/voxbench/issues/24) | Add a framework-independent runtime launcher for cascade test calls | #17, #18, #22 | + +S0 stabilizes the existing uncommitted UI/SSE candidate. It is an additional +prerequisite only if C4 integrates that optional agent dispatcher; C1/common +observation can proceed independently. + +## First work and remaining choices + +C1/#17, C2/#18 and C3A/#19 are implemented as local candidates. See +[the v2 config contract](cascade-config-v2.md) and +[the service observation contract](cascade-observation-api.md), +[the causal analysis contract](cascade-analysis.md), and +[the implementation inventory](implementation-status.md) for checks and precise +scope. V2 execution and audio recording ingestion remain planned; the GitHub +issues remain open until review/publication. S0 and all other slices retain their +previous status. + +Start with #17 for versioned schemas/registry compatibility; develop +#16 independently to stabilize the diagnostic UI candidate. Then +#18 establishes the common metadata-only service observation path; #19 derives +endpoint-backed latency and critical paths from it. + +The next user-designated goal is to create two matched deterministic fake +Cascade test projects, one without middleware and one with Pipecat, then ingest +both into VoxBench for tuning and comparison. This advances #22 and the Pipecat +portion of #23 after #19; it does not select real providers or make Pipecat a +required core dependency. + +Exact provider/model combinations, deployment regions and the second framework +are not selected. These choices belong to adapter/live validation and do not +block common contracts or framework-free fake fixtures. + +Existing packet ingestion and loudness issues remain separate; this plan does not +claim to complete them or silently expand their scope. diff --git a/docs/cascade-observation-api.md b/docs/cascade-observation-api.md new file mode 100644 index 0000000..cff6da5 --- /dev/null +++ b/docs/cascade-observation-api.md @@ -0,0 +1,180 @@ +# Metadata-only service observation API + +Implemented locally for [issue #18](https://github.com/engkimo/voxbench/issues/18). +This contract lets direct applications, Pipecat applications and other runtimes +report the same observed STT/LLM/TTS/realtime boundaries. It does not require a +framework SDK and does not execute providers or retain conversation content. + +## Lifecycle + +1. Resolve a v1 or v2 config. V2 examples are under `examples/v2/`. +2. Start an observation-owned run with `POST /runs/observed`. V2 `/runs` and + `/runs/async` remain unsupported because execution belongs to a later slice. +3. Send batches to `POST /v1/observations` with `run_id` and `service_events`. +4. Finish with `POST /runs/{run_id}/complete` or `/fail`. + +The observation endpoint retains its existing URL because `service_events` is a +backwards-compatible batch extension. Every event carries its own required +`schema_version: voxbench/service-event/v1`. + +## Plain HTTP example + +Any language capable of sending JSON can use the API: + +```json +{ + "run_id": "run UUID returned by /runs/observed", + "service_events": [ + { + "schema_version": "voxbench/service-event/v1", + "collector_alias": "application-a", + "event_alias": "llm-start-1", + "kind": "llm.request_started", + "component_id": "llm-main", + "role": "llm", + "generation_epoch": 0, + "turn_alias": "turn-1", + "request_alias": "llm-request-1", + "parent_request_alias": "stt-request-1", + "response_alias": "response-1", + "attributes": {"attempt_ordinal": 0}, + "ts": "2026-09-27T01:02:03.456+00:00", + "clock_domain": "application-wall", + "alignment_uncertainty_ms": 2.0 + } + ] +} +``` + +The standalone JSON Schema is `schemas/service-event.v1.schema.json`. Registry +semantic checks add component/role/authority validation after structural parsing. + +`collector_alias` namespaces `event_alias`; persisted IDs have the form +`service:{collector_alias}:{event_alias}`. Aliases are application-generated, +run-local safe identifiers. Do not copy provider IDs into them. A timezone-aware +timestamp is required and normalized to UTC. Preserve the actual clock domain +and alignment uncertainty; do not claim clocks are aligned when they are not. + +## Python observer + +```python +from voxbench.observability import ServiceEvent, VoxBenchObserver + +observer = VoxBenchObserver(run_id, transport, max_pending_items=2048) +observer.observe_service_event(ServiceEvent( + collector_alias="application-a", + event_alias="tts-first-1", + kind="tts.first_pcm", + component_id="tts-main", + role="tts", + request_alias="tts-request-1", + response_alias="response-1", + segment_alias="segment-1", + generation_epoch=0, + attributes={"observed_boundary": "first-pcm-frame"}, +)) +observer.flush() # call outside the media callback +``` + +The observer bounds pending items. Items arriving after `max_pending_items` are +dropped rather than blocking a callback; the cumulative count is exposed as +`observation_drop_count`. A failed transport restores the unsent remainder. +Applications should export or alert on the drop counter. Batch limits do not +replace this queue budget. + +## Closed event vocabulary + +| Boundary | Kinds | +| --- | --- | +| Session | `service.session_started`, `service.session_ended`, `service.failed` | +| STT | `stt.partial_emitted`, `stt.final_emitted` | +| Turn | `turn.speech_ended`, `turn.committed` | +| Text LLM | `llm.request_started`, `llm.first_output`, `llm.first_answer_text`, `llm.response_completed` | +| Aggregation | `aggregation.segment_ready` | +| TTS | `tts.request_started`, `tts.first_pcm`, `tts.response_completed` | +| Integrated realtime | `realtime.request_started`, `realtime.first_audio`, `realtime.response_completed` | +| Local playback | `playback.write_started`, `playback.write_completed`, `playback.discarded` | +| Cancellation | `cancel.requested`, `cancel.acknowledged` | + +Kinds and roles are checked together. The server also checks `component_id` +against the resolved run config. STT/LLM/TTS/realtime IDs must match their service +roles; aggregation uses its text processor; playback uses a configured output +PCM stage. `turn.committed` must name the configured end-of-turn authority. +V1 realtime uses the normalized component ID `ai`. + +Observed boundaries retain their literal meaning: + +- `stt.final_emitted` is a bounded STT result, not a committed conversational turn. +- `turn.speech_ended` is an independently observed caller boundary used for STT + finalization and end-to-end local response timing. +- `llm.first_output` records only `answer`, `reasoning`, `tool_call` or `other`; + it does not make reasoning/tool output eligible for speech. +- `llm.first_answer_text` is the first answer boundary; tool/reasoning content is + neither stored nor treated as an answer. +- `tts.first_pcm` is the first observed PCM frame from TTS. +- `playback.write_started` is a local write boundary and does not prove remote + audible playout. +- `cancel.requested` and `cancel.acknowledged` are separate evidence. One does not + imply the other. + +The absence of a terminal event remains incomplete evidence. Completion of the +VoxBench run does not synthesize service terminals. + +## Causal aliases and missing coverage + +Events can carry `session_alias`, `turn_alias`, `request_alias`, +`parent_request_alias`, `response_alias`, `segment_alias`, and a non-negative +`generation_epoch`. Continuous STT uses a session alias; bounded STT final +operations use request and segment aliases. Retries get new request aliases while +remaining associated with the same turn/response as appropriate. + +Each kind requires the relations needed to interpret it. If a required relation +was not observed, omit its alias and list its name in `unobserved_relations`. +This is different from sending an alias whose parent event has not arrived: +unknown parent requests are projected as `unresolved_relations: parent_request`. +That derived marker disappears if the parent later arrives out of order. An +explicit `unobserved_relations` marker remains. + +The payload has no run-reference field inside an event. All relationships are +scoped to the batch `run_id`; attempts to add `run_id`, parent run IDs or other +cross-run fields fail closed. + +## Privacy and scalar attributes + +The DTO forbids extra fields and nested values. It has no transcript, prompt, +generated text, content hash, raw payload, URL, credential, provider ID or tool +argument field. The only optional attributes are allowlisted normalized scalars: + +```text +attempt_ordinal, sequence_ordinal, duration_ms, audio_duration_ms, +input_units, output_units, cached_input_units, tool_call_count, queue_depth, +sample_rate_hz, channels, encoding, retry, truncated, terminal_outcome, +failure_alias, cancel_scope, observed_boundary, output_kind +``` + +Counts and ordinals are non-negative integers, durations are finite and +non-negative, flags are booleans, and strings are short safe aliases. Nulls, +URLs, whitespace-bearing content, containers and unknown keys fail validation. +After causal fields and coverage are normalized, at most 16 scalar attributes +are persisted per event. + +Reserved service kind names cannot be submitted through generic +`timeline_events`; callers must pass the typed validation path. + +## Retry, conflicts and limits + +- Repeating the same collector/event ID with an identical normalized payload is + idempotent. Changing its payload returns HTTP 409. +- Different collectors may use the same event alias without collision. +- Generic timeline retries now follow the same same-payload/conflict rule. +- A batch may contain at most 128 timeline and service events combined. The + Python observer splits larger flushes while preserving unsent data on failure. +- A run accepts at most 10,000 persisted timeline/service events and returns HTTP + 429 before mutation when the limit would be exceeded. +- V2 audio chunks remain unsupported until recording segment maps are added. + +Events use the existing transactional timeline-event persistence. Repository +reload reconstructs the names, component, clock fields, causal aliases, safe +attributes and coverage markers. This establishes event storage, not deterministic +latency derivation. The local issue #19 implementation is documented in +[cascade-analysis.md](cascade-analysis.md). diff --git a/docs/demo-live-softphone.md b/docs/demo-live-softphone.md index 09c6da3..337c9be 100644 --- a/docs/demo-live-softphone.md +++ b/docs/demo-live-softphone.md @@ -1,5 +1,10 @@ # Live Softphone Realtime Demo +This guide describes the integrated realtime/loopback paths. Current feature +and validation status is in [implementation-status.md](implementation-status.md). +The separate STT → LLM → TTS path is a [design proposal](cascade-design.md), +not an available option in the commands below. + This demo is staged as small vertical slices while real SIP/RTP media is wired: ```text diff --git a/docs/diagnostic-agent-design.md b/docs/diagnostic-agent-design.md index dc642c3..b1df098 100644 --- a/docs/diagnostic-agent-design.md +++ b/docs/diagnostic-agent-design.md @@ -1,6 +1,8 @@ # VoxBench 診断エージェント設計・実装計画 v0.1 > Durable decisions and implementation order are summarized in [`MEMORY.md`](../MEMORY.md). +> +> 2026-09-18監査：現在の状態は [implementation status](implementation-status.md) を参照。本書はproduction診断の設計で、§8.1のUI/SSEは未コミットlocal candidate。永続化・LLM・認可が実装済みであることを意味しない。 ## 1. 結論 @@ -38,7 +40,7 @@ VoxBench はすでに診断エージェントの土台を持つ。 ### 2.2 現状の不足 - 診断ルールと API projection の多くが `run_api.py` に集中しており、ツールとして再利用しづらい。 -- Web UI の主要機能が `App.tsx` に集中しており、チャットドロワーをそのまま追加すると保守性が悪化する。 +- Web UI の主要機能が `App.tsx` に集中している。local candidateのagent panelは別componentになったが、Call inspector/dispatcherとAPI projectionの分離は残件。 - raw PCAP/Wireshark import は未実装。現在は構造化 SIP/RTP ingest と一時的 packet tap が中心。 - 一般的な Asterisk/Pipecat/application log の取り込み、全文検索、行単位引用がない。 - source repository の登録、commit snapshot、コード検索、行単位引用がない。 @@ -357,6 +359,30 @@ command type: - commandは再送されても二重再生等を起こさないようidempotentに扱う。 - 自動再生はブラウザのautoplay制約と利用者設定に従う。 +#### Local candidate実装状況（2026-08-12） + +型付きUI commandの最初の実行sliceを実装した。 + +- `POST /runs/{run_id}/ui-commands/resolve` が未信頼proposalを受け取り、commandごとの必須・許可fieldを検証する。 +- run、incident、evidence、比較run、recording stage、artifact、filter値、time windowを保存済みtimelineからserver側で解決する。 +- Web dispatcherが検証済みcommandだけを受け取り、run/incident選択、evidence focus、時間窓、panel、録音、比較、view filterへ反映する。 +- command IDはclientでidempotentに扱い、適用、拒否、autoplay blockを実行結果として表示する。 +- Web右railの`Agent UI command bridge`は、diagnostic orchestrator未実装の間に同じ契約を人間とfake agentで確認するための手動adapterである。 + +#### Local candidateのSSE接続状況（2026-08-15） + +型付きcommandを実際のagent event経路へ接続するend-to-end sliceを追加した。 + +- `POST /runs/{run_id}/diagnostic-sessions`は選択runに限定したlocal deterministic sessionを作る。 +- adapterは最も強いincident、なければ最初のtyped eventを選び、`status`、`evidence_found`、`answer_delta`、`ui_command_proposed`、検証済み`ui_command`、`completed`をordinal付きで生成する。 +- `GET /diagnostic-sessions/{session_id}/events`はSSEとしてeventを配信し、`Last-Event-ID`または`after`から再生できる。 +- Webの`Ask VoxBench` panelがSSEを受信し、検証済みcommandをdispatcherへ渡す。人間がJSONをpasteしなくてもUIが直接移動する。 +- `POST /diagnostic-sessions/{session_id}/ui-command-results`でclient結果をidempotentにacknowledgeし、異なる二重結果は`409`で拒否する。 + +この段階のsession/event/result storeはprocess内memoryであり、外部LLMを呼ばない。質問は保存されるがincident選択には使われず、session作成時に完了した調査のeventをfinite SSEとして返す。serverのordinal replayはWebの自動再接続や新規eventの継続配信を意味しない。session容量/TTL、永続session/event store、production model adapterとbounded orchestrator、long-running job/cancel、`start_guided_sequence`、OIDC認可は未実装。現在のlocal adapterは安全境界とUI配送を確認する未コミットfake/local candidateで、production診断claimを生成するものではない。 + +Cascade診断は [cascade設計案](cascade-design.md) のcomponent/turn/request/segment証拠を同じevidence serviceで扱う。STT/LLM/TTSの会話モデルと、この診断エージェントの`ModelAdapter`は別契約。cascade対応だけでtranscript/audioの診断model送信を許可しない。 + ### 8.2 Guided investigation 回答には、静的な文章とは別に検証済みのstep列を含められる。 @@ -618,8 +644,8 @@ OSS coreにはprovider非依存の`IdentityProvider`/`Authorizer`契約とOIDC 4. deterministic diagnostic bundle endpoint と export。 5. diagnostic session schema/repository/API。 6. fake model + bounded orchestrator + claim validator。 -7. SSE と Ask VoxBench drawer。 -8. 型付きUI command、evidence navigation、guided investigation。 +7. 永続event/jobをSSEとlocal Ask VoxBench candidateへ接続し、cancel/reconnectを実装。 +8. 型付きUI command/evidence navigation candidateの安定化とguided investigation。 9. OIDC operator authとrole/authorization hook。 10. production model adapter、egress policy、運用メトリクス。 diff --git a/docs/implementation-status.md b/docs/implementation-status.md new file mode 100644 index 0000000..fd5c829 --- /dev/null +++ b/docs/implementation-status.md @@ -0,0 +1,274 @@ +# VoxBench implementation status + +Last audited: 2026-09-27. This inventory replaces dated pending lists as the entry +point for current implementation status. It does not turn a local candidate into +a published feature. + +## Baseline and status vocabulary + +The publication worktree baseline is `ae575b6` (`Detect click/pop discontinuities +with bounded PCM evidence`). +Use `git rev-parse HEAD` and `git status --short` to check whether that baseline +has changed; this document is an audit record, not a live Git query. + +| Status | Meaning | +| --- | --- | +| Committed | Implementation exists in the audited Git baseline | +| Local candidate | Implementation exists locally, with verification recorded below, but is absent from the baseline | +| Planned | Design exists; runnable implementation is absent | +| Deployment validation pending | A code boundary exists, but the named deployment behavior has not been established by this audit | + +Product decisions belong in `MEMORY.md`; architecture belongs in `DESIGN.md` and +feature designs. Dated progress entries and Serena implementation memories retain +their historical meaning. Never use an old percentage or pending list as a +current readiness assessment. + +The framework-independent realtime/cascade plan is tracked by +[parent issue #15](https://github.com/engkimo/voxbench/issues/15) and the +[issue dependency plan](cascade-issue-plan.md). C1, C2 and C3A are local candidates; +other tasks remain planned. The +UI/SSE stabilization issue #16 is separate from cascade contract issue #17. + +## Current inventory + +| Area | Status | Evidence and remaining boundary | +| --- | --- | --- | +| Config/manifest schemas, overlays, resolved hashes, static validation | Committed | `schemas.py`, `registry/`; currently one `ai.provider/model`, v1 only | +| V2 realtime/cascade config and versioned capabilities | Local candidate | `schemas_v2.py`, `registry/v2.py`, `registry/config_views.py`, `test_registry_v2.py`; role/format/ID/authority checks, manifest pins, same-mode overlays; [contract guide](cascade-config-v2.md) | +| Framework-independent service event observation | Local candidate | `observability/service_events.py`, observer/HTTP batch extension and `test_service_observations.py`; v1/v2, persistence, causal aliases, privacy, idempotency/conflicts, bounded queues/run events; [HTTP contract](cascade-observation-api.md) | +| Cascade causal grouping and latency analysis | Local candidate | `observability/cascade_analysis.py`, safe run component projections and `test_cascade_analysis.py`; explicit parent chains, endpoint evidence, critical path, clock/missing states and opt-in SLOs; [analysis contract](cascade-analysis.md) | +| Harness WAV taps and OTel spans | Committed | `engine_harness/`; the default `run_once` generates nominal artifacts, not a real provider conversation | +| Signal invariants and synthetic full-reference scoring | Committed | `verification/`, `synthetic_caller/`; optional ViSQOL CLI, explicit blocked/unavailable states, aggregation/regression/calibration | +| Common timeline and deterministic incidents | Committed | `run_api.py`, `tests/test_timeline_stage_diagnostics.py`; five primitives, clock uncertainty, explicit observation boundaries | +| Stage playback, waveform, two-run comparison, deep links | Committed | `web/src/App.tsx`, `web/src/types.ts` | +| Observed-run library integration | Committed | `observability/observer.py`; framework-independent Python/HTTP path, existing applications keep pipeline ownership | +| Pipecat helper and other-framework adapters | Helper committed; dedicated adapters planned | `engine_harness/pipecat_adapter.py` only wraps pipeline construction; Pipecat is not a required package, other middleware uses application instrumentation today | +| RTP fixed-header tap and capture-health accounting | Committed | `RtpPacketTapAdapter`, observer and timeline tests; owner must supply actual drop counters | +| Caller VAD, local playback, barge-in and queue-discard evidence | Committed | `telephony/audiosocket.py`; local write/disposal is not remote audible playout | +| Asterisk AudioSocket and Gemini/OpenAI realtime adapters | Committed | `telephony/`, `realtime_providers/`; production timing/auth/TLS remain deployment work | +| Automatic per-call AMI RTCP collection and matched-condition tags | Committed | `ff65ada` / PR #8; aggregate evidence, one active local test call for safe attribution | +| Cross-session resource trend detector | Committed | `/runs/cross-session-trends`, `tests/test_cross_session_trends.py`; three or more ended runs on the same environment/server | +| Postgres run persistence and leased/fenced async worker | Committed | repository, job queue, worker, migrations; multi-process deployment soak remains pending | +| MinIO sink and bounded authenticated remote playback | Committed | storage/audio-session boundaries; remote audio cookie is not product-wide operator authentication | +| Typed UI-command resolver and Web dispatcher | Local candidate | Exact files below; nine command types, run/evidence target resolution and client idempotence | +| Deterministic diagnostic sessions, finite SSE replay and result acknowledgement | Local candidate | Exact files below; process-local store, one selected incident/event, no external model | +| Session/event/result persistence, model orchestration, diagnostic jobs/cancel, guided sequence | Planned | `diagnostic-agent-design.md`; not completed by the local SSE slice | +| Product-wide OIDC authentication, authorization, audit/retention | Planned | `diagnostic-agent-design.md`; required before production diagnostic integration | +| Cascade recording maps, inspector, test projects, adapters and runtime | Planned | `cascade-design.md`; causal analysis is local, while v2 audio chunks and execution remain explicitly unsupported. Next goal: matched direct/Pipecat fake Cascade projects for tuning and evaluation | +| Raw PCAP import, log/source snapshot connectors, second engine, scale profile | Planned | Designs/roadmap; not established by the RTP tap or second realtime provider | + +## Local UI/SSE candidate audit + +The local candidate is self-contained only when all these implementation files +are included. Four existing implementation files are modified: + +- `src/voxbench/control_plane/run_api.py` +- `web/src/App.tsx` +- `web/src/styles.css` +- `web/src/types.ts` + +Eight required implementation/test files are currently untracked: + +- `src/voxbench/control_plane/ui_commands.py` +- `src/voxbench/control_plane/diagnostic_sessions.py` +- `tests/test_ui_commands.py` +- `tests/test_diagnostic_sessions.py` +- `web/src/AgentUiCommandPanel.tsx` +- `web/src/AgentDiagnosticPanel.tsx` +- `web/src/uiCommands.ts` +- `web/src/diagnosticSessions.ts` + +Related documentation changes already exist in `MEMORY.md`, `README.md`, and +`docs/diagnostic-agent-design.md`. They must accompany stabilization. Current +tracked code imports untracked modules; copying only tracked changes would +produce an incomplete application. + +Verified local endpoints: + +```text +POST /runs/{run_id}/ui-commands/resolve +POST /runs/{run_id}/diagnostic-sessions +GET /diagnostic-sessions/{session_id} +GET /diagnostic-sessions/{session_id}/events +POST /diagnostic-sessions/{session_id}/ui-command-results +``` + +The deterministic planner selects the highest-severity incident, then the +earliest typed event, then an available panel. The submitted question is stored, +but does not change that selection. Session creation finishes the investigation +synchronously. SSE returns a finite snapshot of already-created events and +supports ordinal replay via `Last-Event-ID`/`after`; it is not a long-running +stream of newly arriving investigation events. The Web panel closes on +`completed` or stream error. Server replay support does not establish automatic +client reconnection. Acknowledgements are process-local and conflicting duplicate +results return `409`. + +The local store has no session TTL/capacity eviction. Session references and +typed targets are validated, but product-wide operator authorization is absent. +Comparison currently checks run existence, not a production authorizer's grant. +This candidate cannot be described as a production diagnostic agent. + +## Verification recorded on 2026-09-18 + +| Check | Result | Scope | +| --- | --- | --- | +| `.venv/bin/ruff check .` | Passed | Current local Python source | +| `.venv/bin/pytest -q` | 330 passed, 4 skipped | Python 3.14.3; four opt-in Postgres tests skipped; one Starlette/httpx deprecation warning | +| `npm --prefix web run build` | Passed | TypeScript and Vite production build | + +The same checks also passed in a temporary checkout assembled from +`git archive ef7deec` plus exactly the twelve implementation/test files listed +above: 330 passed / 4 skipped, Ruff passed, Web build passed. Python packages and +Web dependencies were reused from the installed development environment; the +application source came from that isolated checkout. Personal files and other +untracked implementation files were not included. This establishes the candidate's +source-file completeness, not a clean dependency install or published feature. + +This audit did not validate a browser interaction, actual STT/TTS/provider call, +new Postgres connectivity, remote MinIO deployment, OIDC login, or published CI. +Historical real-call evidence is not a substitute for those checks. + +## Separate UI/SSE stabilization plan + +1. Review the twelve implementation/test files as a separate existing change. + Keep the documentation-consistency/cascade design change independently + reviewable. Include every imported new module; exclude personal local files. +2. Isolated source-file verification is complete for the audited candidate: + the exact twelve-file set passed Python checks and Web build on top of the + committed baseline. Before publication, repeat with the final reviewed file + set and freshly installed dependencies in the supported CI environment. +3. Check the UI path in a browser: incident/event fallback, shared cursor, + scope rejection, duplicate command, playback/autoplay result, SSE replay and + acknowledgement. Use synthetic evidence. +4. Record remaining runtime limits explicitly. Session capacity/TTL, Web races + and reconnection, authorization, and durable diagnostic work each need their + own acceptance criteria before production use. +5. Commit the reviewed change as its own unit when implementation stabilization + is requested, then update this inventory with its commit and verification. + C1 implementation has not committed or published the UI/SSE candidate. + +Cascade contracts are implemented against the committed v1 contracts independently +of that step. The local UI dispatcher is an optional integration candidate with a +separate stabilization dependency, rather than an implicit released dependency. + +## C1 config/capability local candidate (2026-09-18) + +Scope: [#17](https://github.com/engkimo/voxbench/issues/17), implemented locally, +not committed or published. The issue remains open for review/publication. +[cascade-config-v2.md](cascade-config-v2.md) records the implemented contract. + +Required new files: + +- `src/voxbench/schemas_v2.py` +- `src/voxbench/registry/v2.py` +- `src/voxbench/registry/config_views.py` +- `tests/test_registry_v2.py` +- `tests/fixtures/configs/v1-baseline.resolved.json` +- `tests/fixtures/configs/v1-overlay.resolved.json` +- `schemas/config.v2.schema.json`, `schemas/manifest.v2.schema.json` +- The two config and eight manifest JSON documents under `examples/v2/` +- `docs/cascade-config-v2.md` and the current status/decision references + +Required existing-source changes: `registry/service.py`, `schemas_export.py`, +`engine_harness/plan.py`, `engine_harness/harness.py`, +`synthetic_caller/offline.py`, `verification/core.py`, and only the v2 rejection +guard/import in `control_plane/run_api.py`. That API file also contains the +independent pre-existing UI/SSE candidate; publishing its entire working-tree +diff would combine scopes. + +| Check | Final result | Scope | +| --- | --- | --- | +| Current-tree Ruff and full Python suite | Passed; 408 passed / 4 skipped | Includes the independent UI/SSE candidate and 78 C1 acceptance cases | +| Isolated `git archive ef7deec` plus C1 only | Ruff passed; 401 passed / 4 skipped | Excludes untracked UI/SSE modules/tests and all personal files; only C1 API guard was applied to the committed API source | +| V1 contract files | Byte-identical to HEAD | `schemas.py`, `config.schema.json`, `manifest.schema.json`; frozen baseline/overlay JSON and hashes pass | +| CLI examples | Realtime and cascade passed | Static resolution without middleware or provider calls; four/seven manifest pins respectively | +| Documentation links and diff whitespace | Passed | Current design/status/contract documents | + +Both checkouts use Python 3.14.3 and existing installed dependencies; this is +source-completeness verification, not a fresh dependency install or published +CI. Four opt-in Postgres tests are skipped and one Starlette/httpx deprecation +warning remains. Stored-v1 roundtrip uses the SQLAlchemy repository with SQLite. +No browser or real provider call is needed for C1 and none was run. Web sources +are unchanged by this slice; their earlier build belongs to the UI/SSE audit. + +At C1 completion, v2 supported static config resolution only. C2 now permits +observation-only run creation and safe typed service-event ingest. Legacy harness, +synthetic generation, audio verification and execution APIs still fail before +output. C3A now derives causal timing; recording mapping, UI, framework/provider +test projects/adapters and execution remain separate future slices. Validation does not establish live +provider/model readiness or complete observation coverage. + +## C2 service-observation local candidate (2026-09-27) + +Scope: [#18](https://github.com/engkimo/voxbench/issues/18), implemented locally, +not committed or published. The issue remains open for review/publication. +[cascade-observation-api.md](cascade-observation-api.md) is the normative HTTP, +privacy, identity and retry guide. + +Required new files: + +- `src/voxbench/observability/service_events.py` +- `tests/test_service_observations.py` (25 focused acceptance cases) +- `schemas/service-event.v1.schema.json` +- `docs/cascade-observation-api.md` + +Required existing-source changes: `observability/observer.py` and `__init__.py`, +`schemas_export.py`, C2 endpoint/normalization changes in +`control_plane/run_api.py`, and the changed observation expectation in +`test_registry_v2.py`. The API file also contains the independent UI/SSE +candidate; the isolated verification applies only C1/C2 diff hunks to HEAD. + +| Check | Final result | Scope | +| --- | --- | --- | +| Current-tree Ruff and full Python suite | Passed; 433 passed / 4 skipped | Includes UI/SSE, C1 and 25 C2 focused tests | +| Isolated `git archive ef7deec` plus C1+C2 | Ruff passed; 426 passed / 4 skipped | UI/SSE modules/tests and personal files excluded; imports resolve from isolated source | +| Plain HTTP and Python observer | Passed | Multi-turn/multi-segment cascade, v1/v2 realtime, no framework SDK | +| Persistence/reconstruction | Passed with SQLAlchemy SQLite | Safe event fields and derived missing-parent coverage survive repository reload | +| Privacy/identity/retry limits | Passed | Extra content/raw/cross-run fields fail; exact retry succeeds; conflict 409; collectors namespaced; combined batch/queue/run bounds | +| Service-event JSON Schema | Generated and validated | Standalone draft 2020-12 artifact matches exported model | + +Both runs use Python 3.14.3 and existing installed dependencies; this remains +source-completeness verification rather than a fresh dependency install or +published CI. Four opt-in Postgres tests are skipped and the existing +Starlette/httpx deprecation warning remains. No provider call, browser run or +actual Pipecat/other-framework integration was performed. C2 normalizes observed +metadata; C3A now derives service latency, while audio segment mapping, remote +playout and provider readiness remain unestablished. + +## C3A causal-analysis local candidate (2026-09-27) + +Scope: [#19](https://github.com/engkimo/voxbench/issues/19), implemented locally, +not committed or published. The issue remains open for review/publication. +[cascade-analysis.md](cascade-analysis.md) records the measurement, clock, +critical-path, coverage and SLO contract. + +Required new files: + +- `src/voxbench/observability/cascade_analysis.py` +- `tests/test_cascade_analysis.py` (10 focused acceptance cases) +- `docs/cascade-analysis.md` + +Required existing-source changes: two additional closed service boundaries and +`output_kind` validation in `observability/service_events.py`; C3 projections, +intervals and SLO incidents in `control_plane/run_api.py`; safe component +projection in `registry/config_views.py`; v2-only explicit latency SLO contracts +in `schemas_v2.py`; generated v2/service-event schemas; and an opt-in real +Postgres restart case in `test_postgres_integration.py`. + +| Check | Final result | Scope | +| --- | --- | --- | +| Publication-tree Ruff and full Python suite | Passed; 444 passed / 5 skipped | `ae575b6` plus C1–C3; includes the committed click/pop coverage and 10 C3 focused tests | +| Isolated `git archive ef7deec` plus C1–C3 | Ruff passed; 436 passed / 5 skipped | UI/SSE modules/tests and personal files excluded; imports resolve from isolated source | +| Multi-segment critical path | Passed | End-to-end local write wait uses linked first segment; overlapping lifecycle durations are not added | +| Answer/tool/retry/epoch grouping | Passed | First output differs from first spoken answer; continuations are separate operations; old playback is not reused | +| Missing/clock evidence | Passed | Missing boundary is unobserved; unmatched parent stays explicit; cross-domain/negative time is indeterminate | +| Persistence/reconstruction | Passed with SQLAlchemy SQLite | Safe components, grouping, measurements and evidence refs reproduce after repository reload | +| Explicit latency SLO | Passed | Incident exists only for an observed applicable configured threshold; no default quality/latency inference | +| Real Postgres restart | Test added; skipped locally | Requires `VOXBENCH_TEST_POSTGRES_URL`; local run did not claim a Postgres deployment result | + +Both runs use Python 3.12.3 and existing installed dependencies. Five opt-in +Postgres tests are skipped and the existing Starlette/httpx warning remains. +No provider call, browser run, Pipecat runtime or other framework was exercised. +The next user-designated goal is to build matched deterministic Cascade test +projects with and without Pipecat, feed both through this common contract, and +use VoxBench to compare, tune and evaluate them. diff --git a/docs/library-integration.md b/docs/library-integration.md index c9f8445..1f2c9b8 100644 --- a/docs/library-integration.md +++ b/docs/library-integration.md @@ -18,6 +18,22 @@ The public boundary is `voxbench.observability`. It has no OpenAI, Gemini, Pipec or HTTP client package dependency. `HttpObservationTransport` uses the Python standard library. +Current availability is recorded in [implementation-status.md](implementation-status.md). +Native metadata-only observation of separately configured STT/LLM/TTS services +is implemented as a local candidate in +[cascade-observation-api.md](cascade-observation-api.md). Deterministic endpoint +analysis is also a local candidate in [cascade-analysis.md](cascade-analysis.md). +V2 observation-only runs establish native cascade config, service correlation +and analysis, while execution and v2 recording mapping remain unsupported. + +Pipecat is optional. The observer and HTTP ingest can be instrumented from another +framework or a custom application with no middleware. The current Pipecat helper +is a thin construction boundary, not a required execution engine. Dedicated +adapters and executable direct/Pipecat test projects are future work. Their target +contract is the same metadata-only observation API and causal analysis, with a +shared conformance suite and a no-middleware reference example. Non-Python +applications can use the HTTP contract without a Python SDK. + ## Lifecycle 1. Start an observed run with the same resolved-config inputs used by normal diff --git a/docs/provider-adapter-guide.md b/docs/provider-adapter-guide.md index deede63..dd38ce3 100644 --- a/docs/provider-adapter-guide.md +++ b/docs/provider-adapter-guide.md @@ -5,6 +5,14 @@ provider-neutral stream of PCM audio and lifecycle evidence. They do not move provider-specific response bodies, URLs, credentials, or identifiers into the core timeline. +This guide covers the committed integrated realtime adapter boundary. A common +metadata-only STT/text-LLM/TTS observation contract is a local candidate in +[cascade-observation-api.md](cascade-observation-api.md), with deterministic +endpoint analysis in [cascade-analysis.md](cascade-analysis.md); provider-specific +service adapters are not implemented yet. Keep `RealtimeProviderSession` compatible; do not +force text or STT/TTS lifecycle through its PCM-only interface. Current feature +availability is in [implementation-status.md](implementation-status.md). + ## Adapter boundary Implement the protocols in diff --git a/examples/v2/configs/cascade.json b/examples/v2/configs/cascade.json new file mode 100644 index 0000000..4037aa4 --- /dev/null +++ b/examples/v2/configs/cascade.json @@ -0,0 +1,90 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "VoiceConfig", + "meta": { + "name": "example-cascade", + "version": "1.0.0" + }, + "spec": { + "engine": { + "kind": "example-telephony", + "manifest_version": "1.0.0", + "params": {} + }, + "transport": { + "codec": "mulaw", + "ptime_ms": 20 + }, + "media": { + "input_pipeline": [ + { + "id": "input-resampler", + "type": "resampler", + "plugin": "example-input-resampler", + "manifest_version": "1.0.0", + "params": {}, + "host_capabilities": [ + "cadence_pacing" + ] + } + ], + "output_pipeline": [ + { + "id": "output-resampler", + "type": "resampler", + "plugin": "example-output-resampler", + "manifest_version": "1.0.0", + "params": {}, + "host_capabilities": [ + "cadence_pacing" + ] + } + ] + }, + "turn_taking": { + "speech_detector": "application", + "end_of_turn": "application", + "interruption": "application" + }, + "ai": { + "mode": "cascade", + "stt": { + "id": "stt-main", + "plugin": "example-stt", + "manifest_version": "1.0.0", + "model": "pinned-stt-model", + "params": {}, + "input_audio": { + "encoding": "pcm16", + "rate": 16000, + "channels": 1 + } + }, + "llm": { + "id": "llm-main", + "plugin": "example-llm", + "manifest_version": "1.0.0", + "model": "pinned-llm-model", + "params": {} + }, + "tts": { + "id": "tts-main", + "plugin": "example-tts", + "manifest_version": "1.0.0", + "model": "pinned-tts-model", + "params": {}, + "output_audio": { + "encoding": "pcm16", + "rate": 24000, + "channels": 1 + } + }, + "text_aggregation": { + "id": "aggregation-main", + "plugin": "example-sentence-aggregator", + "manifest_version": "1.0.0", + "params": {} + } + } + } +} diff --git a/examples/v2/configs/realtime.json b/examples/v2/configs/realtime.json new file mode 100644 index 0000000..2ea7a50 --- /dev/null +++ b/examples/v2/configs/realtime.json @@ -0,0 +1,70 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "VoiceConfig", + "meta": { + "name": "example-realtime", + "version": "1.0.0" + }, + "spec": { + "engine": { + "kind": "example-telephony", + "manifest_version": "1.0.0", + "params": {} + }, + "transport": { + "codec": "mulaw", + "ptime_ms": 20 + }, + "media": { + "input_pipeline": [ + { + "id": "input-resampler", + "type": "resampler", + "plugin": "example-input-resampler", + "manifest_version": "1.0.0", + "params": {}, + "host_capabilities": [ + "cadence_pacing" + ] + } + ], + "output_pipeline": [ + { + "id": "output-resampler", + "type": "resampler", + "plugin": "example-output-resampler", + "manifest_version": "1.0.0", + "params": {}, + "host_capabilities": [ + "cadence_pacing" + ] + } + ] + }, + "turn_taking": { + "speech_detector": "application", + "end_of_turn": "application", + "interruption": "application" + }, + "ai": { + "mode": "realtime", + "realtime": { + "id": "realtime-main", + "plugin": "example-realtime", + "manifest_version": "1.0.0", + "model": "pinned-realtime-model", + "params": {}, + "input_audio": { + "encoding": "pcm16", + "rate": 16000, + "channels": 1 + }, + "output_audio": { + "encoding": "pcm16", + "rate": 24000, + "channels": 1 + } + } + } + } +} diff --git a/examples/v2/manifests/aggregation.json b/examples/v2/manifests/aggregation.json new file mode 100644 index 0000000..ebfcdac --- /dev/null +++ b/examples/v2/manifests/aggregation.json @@ -0,0 +1,14 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "processor", + "name": "example-sentence-aggregator", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "text_io": { + "input_modality": "text", + "output_modality": "text" + } +} diff --git a/examples/v2/manifests/engine.json b/examples/v2/manifests/engine.json new file mode 100644 index 0000000..238ae30 --- /dev/null +++ b/examples/v2/manifests/engine.json @@ -0,0 +1,26 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "engine", + "name": "example-telephony", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "engine_caps": { + "supported_codecs": [ + "mulaw", + "pcm16" + ], + "decoded_input": { + "encoding": "pcm16", + "rate": 8000, + "channels": 1 + }, + "decoded_output": { + "encoding": "pcm16", + "rate": 8000, + "channels": 1 + } + } +} diff --git a/examples/v2/manifests/input-resampler.json b/examples/v2/manifests/input-resampler.json new file mode 100644 index 0000000..d661546 --- /dev/null +++ b/examples/v2/manifests/input-resampler.json @@ -0,0 +1,39 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "processor", + "name": "example-input-resampler", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "io": { + "mode": "rate_changing", + "accepts": { + "encoding": "pcm16", + "rate": 8000, + "channels": 1 + }, + "produces": { + "encoding": "pcm16", + "rate": 16000, + "channels": 1 + } + }, + "requires_host_capability": [ + "cadence_pacing" + ], + "invariants_enforced": [ + "duration_preserving" + ], + "invariants_applicable": [ + "duration_preserving", + "isochronous" + ], + "allowed_overrides": [ + { + "target": "quality", + "type": "string" + } + ] +} diff --git a/examples/v2/manifests/llm.json b/examples/v2/manifests/llm.json new file mode 100644 index 0000000..a8b5eef --- /dev/null +++ b/examples/v2/manifests/llm.json @@ -0,0 +1,29 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "provider", + "name": "example-llm", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "service_caps": { + "roles": [ + "llm" + ], + "input_modality": "text", + "output_modality": "text", + "lifecycle": { + "partial_output": true, + "final_output": true, + "response_boundaries": true + }, + "cancellation": { + "request": true, + "terminal_ack": true + }, + "supported_models": [ + "pinned-llm-model" + ] + } +} diff --git a/examples/v2/manifests/output-resampler.json b/examples/v2/manifests/output-resampler.json new file mode 100644 index 0000000..970eea6 --- /dev/null +++ b/examples/v2/manifests/output-resampler.json @@ -0,0 +1,39 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "processor", + "name": "example-output-resampler", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "io": { + "mode": "rate_changing", + "accepts": { + "encoding": "pcm16", + "rate": 24000, + "channels": 1 + }, + "produces": { + "encoding": "pcm16", + "rate": 8000, + "channels": 1 + } + }, + "requires_host_capability": [ + "cadence_pacing" + ], + "invariants_enforced": [ + "duration_preserving" + ], + "invariants_applicable": [ + "duration_preserving", + "isochronous" + ], + "allowed_overrides": [ + { + "target": "quality", + "type": "string" + } + ] +} diff --git a/examples/v2/manifests/realtime.json b/examples/v2/manifests/realtime.json new file mode 100644 index 0000000..e7e2fc7 --- /dev/null +++ b/examples/v2/manifests/realtime.json @@ -0,0 +1,48 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "provider", + "name": "example-realtime", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "service_caps": { + "roles": [ + "realtime" + ], + "input_modality": "audio", + "output_modality": "audio", + "lifecycle": { + "partial_output": true, + "final_output": true, + "response_boundaries": true + }, + "cancellation": { + "request": true, + "terminal_ack": true + }, + "audio_inputs": [ + { + "encoding": "pcm16", + "rate": 16000, + "channels": 1 + } + ], + "audio_outputs": [ + { + "encoding": "pcm16", + "rate": 24000, + "channels": 1 + } + ], + "authorities": [ + "speech_detector", + "end_of_turn", + "interruption" + ], + "supported_models": [ + "pinned-realtime-model" + ] + } +} diff --git a/examples/v2/manifests/stt.json b/examples/v2/manifests/stt.json new file mode 100644 index 0000000..e02dfc1 --- /dev/null +++ b/examples/v2/manifests/stt.json @@ -0,0 +1,41 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "provider", + "name": "example-stt", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "service_caps": { + "roles": [ + "stt" + ], + "input_modality": "audio", + "output_modality": "text", + "lifecycle": { + "partial_output": true, + "final_output": true, + "response_boundaries": true + }, + "cancellation": { + "request": true, + "terminal_ack": true + }, + "audio_inputs": [ + { + "encoding": "pcm16", + "rate": 16000, + "channels": 1 + } + ], + "authorities": [ + "speech_detector", + "end_of_turn", + "interruption" + ], + "supported_models": [ + "pinned-stt-model" + ] + } +} diff --git a/examples/v2/manifests/tts.json b/examples/v2/manifests/tts.json new file mode 100644 index 0000000..348eaa9 --- /dev/null +++ b/examples/v2/manifests/tts.json @@ -0,0 +1,36 @@ +{ + "apiVersion": "voxbench/v2", + "kind": "provider", + "name": "example-tts", + "version": "1.0.0", + "param_schema": { + "type": "object", + "additionalProperties": false + }, + "service_caps": { + "roles": [ + "tts" + ], + "input_modality": "text", + "output_modality": "audio", + "lifecycle": { + "partial_output": true, + "final_output": true, + "response_boundaries": true + }, + "cancellation": { + "request": true, + "terminal_ack": true + }, + "audio_outputs": [ + { + "encoding": "pcm16", + "rate": 24000, + "channels": 1 + } + ], + "supported_models": [ + "pinned-tts-model" + ] + } +} diff --git a/schemas/config.v2.schema.json b/schemas/config.v2.schema.json new file mode 100644 index 0000000..24ae4d5 --- /dev/null +++ b/schemas/config.v2.schema.json @@ -0,0 +1,769 @@ +{ + "$defs": { + "AudioFormat": { + "additionalProperties": false, + "properties": { + "channels": { + "exclusiveMinimum": 0, + "title": "Channels", + "type": "integer" + }, + "encoding": { + "enum": [ + "pcm16", + "pcm24", + "pcm32", + "float32" + ], + "title": "Encoding", + "type": "string" + }, + "rate": { + "exclusiveMinimum": 0, + "title": "Rate", + "type": "integer" + } + }, + "required": [ + "encoding", + "rate", + "channels" + ], + "title": "AudioFormat", + "type": "object" + }, + "AudioIoContract": { + "additionalProperties": false, + "properties": { + "accepts": { + "$ref": "#/$defs/AudioFormat" + }, + "mode": { + "enum": [ + "passthrough", + "rate_changing", + "format_changing" + ], + "title": "Mode", + "type": "string" + }, + "produces": { + "$ref": "#/$defs/AudioFormat" + } + }, + "required": [ + "mode", + "accepts", + "produces" + ], + "title": "AudioIoContract", + "type": "object" + }, + "CascadeAiConfig": { + "additionalProperties": false, + "properties": { + "llm": { + "$ref": "#/$defs/ConversationServiceConfig" + }, + "mode": { + "const": "cascade", + "title": "Mode", + "type": "string" + }, + "stt": { + "$ref": "#/$defs/ServiceConfig" + }, + "text_aggregation": { + "$ref": "#/$defs/ComponentConfig" + }, + "tts": { + "$ref": "#/$defs/ServiceConfig" + } + }, + "required": [ + "mode", + "stt", + "llm", + "text_aggregation", + "tts" + ], + "title": "CascadeAiConfig", + "type": "object" + }, + "ComponentConfig": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Id", + "type": "string" + }, + "manifest_version": { + "minLength": 1, + "title": "Manifest Version", + "type": "string" + }, + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + }, + "plugin": { + "minLength": 1, + "title": "Plugin", + "type": "string" + } + }, + "required": [ + "id", + "plugin", + "manifest_version" + ], + "title": "ComponentConfig", + "type": "object" + }, + "ConfigMeta": { + "additionalProperties": false, + "properties": { + "labels": { + "additionalProperties": true, + "title": "Labels", + "type": "object" + }, + "name": { + "title": "Name", + "type": "string" + }, + "parent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "version" + ], + "title": "ConfigMeta", + "type": "object" + }, + "ConversationServiceConfig": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Id", + "type": "string" + }, + "input_audio": { + "anyOf": [ + { + "$ref": "#/$defs/AudioFormat" + }, + { + "type": "null" + } + ], + "default": null + }, + "manifest_version": { + "minLength": 1, + "title": "Manifest Version", + "type": "string" + }, + "model": { + "minLength": 1, + "title": "Model", + "type": "string" + }, + "output_audio": { + "anyOf": [ + { + "$ref": "#/$defs/AudioFormat" + }, + { + "type": "null" + } + ], + "default": null + }, + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + }, + "plugin": { + "minLength": 1, + "title": "Plugin", + "type": "string" + }, + "system_prompt_ref": { + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "System Prompt Ref" + }, + "tools": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Tools", + "type": "array" + } + }, + "required": [ + "id", + "plugin", + "manifest_version", + "model" + ], + "title": "ConversationServiceConfig", + "type": "object" + }, + "EngineConfigV2": { + "additionalProperties": false, + "properties": { + "kind": { + "minLength": 1, + "title": "Kind", + "type": "string" + }, + "manifest_version": { + "minLength": 1, + "title": "Manifest Version", + "type": "string" + }, + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + } + }, + "required": [ + "kind", + "manifest_version" + ], + "title": "EngineConfigV2", + "type": "object" + }, + "JitterBufferConfig": { + "additionalProperties": false, + "properties": { + "max_ms": { + "title": "Max Ms", + "type": "integer" + }, + "mode": { + "title": "Mode", + "type": "string" + } + }, + "required": [ + "mode", + "max_ms" + ], + "title": "JitterBufferConfig", + "type": "object" + }, + "MediaConfigV2": { + "additionalProperties": false, + "properties": { + "input_pipeline": { + "items": { + "$ref": "#/$defs/PipelineStageV2" + }, + "title": "Input Pipeline", + "type": "array" + }, + "output_pipeline": { + "items": { + "$ref": "#/$defs/PipelineStageV2" + }, + "title": "Output Pipeline", + "type": "array" + } + }, + "required": [ + "input_pipeline", + "output_pipeline" + ], + "title": "MediaConfigV2", + "type": "object" + }, + "ObservabilityConfigV2": { + "additionalProperties": false, + "properties": { + "cross_session": { + "items": { + "type": "string" + }, + "title": "Cross Session", + "type": "array" + }, + "host_metrics": { + "items": { + "type": "string" + }, + "title": "Host Metrics", + "type": "array" + }, + "record_audio": { + "default": false, + "title": "Record Audio", + "type": "boolean" + }, + "service_latency_slos": { + "items": { + "$ref": "#/$defs/ServiceLatencySloV2" + }, + "title": "Service Latency Slos", + "type": "array" + }, + "signal_metrics": { + "items": { + "type": "string" + }, + "title": "Signal Metrics", + "type": "array" + }, + "stage_taps": { + "default": false, + "title": "Stage Taps", + "type": "boolean" + }, + "trace_sample": { + "default": 0.0, + "title": "Trace Sample", + "type": "number" + } + }, + "title": "ObservabilityConfigV2", + "type": "object" + }, + "OverrideValue": { + "additionalProperties": false, + "properties": { + "target": { + "title": "Target", + "type": "string" + }, + "value": { + "title": "Value" + } + }, + "required": [ + "target", + "value" + ], + "title": "OverrideValue", + "type": "object" + }, + "PipelineStageV2": { + "additionalProperties": false, + "properties": { + "host_capabilities": { + "items": { + "type": "string" + }, + "title": "Host Capabilities", + "type": "array" + }, + "id": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Id", + "type": "string" + }, + "invariants_applicable": { + "anyOf": [ + { + "items": { + "enum": [ + "duration_preserving", + "level_preserving", + "isochronous" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Invariants Applicable" + }, + "invariants_enforced": { + "anyOf": [ + { + "items": { + "enum": [ + "duration_preserving", + "level_preserving", + "isochronous" + ], + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Invariants Enforced" + }, + "io": { + "anyOf": [ + { + "$ref": "#/$defs/AudioIoContract" + }, + { + "type": "null" + } + ], + "default": null + }, + "lossy_expected": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lossy Expected" + }, + "manifest_version": { + "minLength": 1, + "title": "Manifest Version", + "type": "string" + }, + "overrides": { + "items": { + "$ref": "#/$defs/OverrideValue" + }, + "title": "Overrides", + "type": "array" + }, + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + }, + "plugin": { + "minLength": 1, + "title": "Plugin", + "type": "string" + }, + "requires_host_capability": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Requires Host Capability" + }, + "type": { + "minLength": 1, + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "plugin", + "manifest_version", + "type" + ], + "title": "PipelineStageV2", + "type": "object" + }, + "RealtimeAiConfig": { + "additionalProperties": false, + "properties": { + "mode": { + "const": "realtime", + "title": "Mode", + "type": "string" + }, + "realtime": { + "$ref": "#/$defs/ConversationServiceConfig" + } + }, + "required": [ + "mode", + "realtime" + ], + "title": "RealtimeAiConfig", + "type": "object" + }, + "ServiceConfig": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Id", + "type": "string" + }, + "input_audio": { + "anyOf": [ + { + "$ref": "#/$defs/AudioFormat" + }, + { + "type": "null" + } + ], + "default": null + }, + "manifest_version": { + "minLength": 1, + "title": "Manifest Version", + "type": "string" + }, + "model": { + "minLength": 1, + "title": "Model", + "type": "string" + }, + "output_audio": { + "anyOf": [ + { + "$ref": "#/$defs/AudioFormat" + }, + { + "type": "null" + } + ], + "default": null + }, + "params": { + "additionalProperties": true, + "title": "Params", + "type": "object" + }, + "plugin": { + "minLength": 1, + "title": "Plugin", + "type": "string" + } + }, + "required": [ + "id", + "plugin", + "manifest_version", + "model" + ], + "title": "ServiceConfig", + "type": "object" + }, + "ServiceLatencySloV2": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Id", + "type": "string" + }, + "max_ms": { + "exclusiveMinimum": 0, + "title": "Max Ms", + "type": "number" + }, + "measurement": { + "enum": [ + "stt_finalization_wait", + "turn_coordination_wait", + "llm_dispatch_wait", + "llm_first_output_wait", + "llm_first_answer_text_wait", + "text_aggregation_wait", + "tts_dispatch_queue_wait", + "tts_first_audio_wait", + "output_playback_start_wait", + "end_to_end_local_response_wait" + ], + "title": "Measurement", + "type": "string" + } + }, + "required": [ + "id", + "measurement", + "max_ms" + ], + "title": "ServiceLatencySloV2", + "type": "object" + }, + "TransportConfig": { + "additionalProperties": false, + "properties": { + "codec": { + "title": "Codec", + "type": "string" + }, + "jitter_buffer": { + "anyOf": [ + { + "$ref": "#/$defs/JitterBufferConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "ptime_ms": { + "title": "Ptime Ms", + "type": "integer" + } + }, + "required": [ + "codec", + "ptime_ms" + ], + "title": "TransportConfig", + "type": "object" + }, + "TurnTakingConfigV2": { + "additionalProperties": false, + "properties": { + "end_of_turn": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "End Of Turn", + "type": "string" + }, + "interruption": { + "anyOf": [ + { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Interruption" + }, + "speech_detector": { + "pattern": "^[a-z][a-z0-9_-]{0,63}$", + "title": "Speech Detector", + "type": "string" + } + }, + "required": [ + "speech_detector", + "end_of_turn" + ], + "title": "TurnTakingConfigV2", + "type": "object" + }, + "VoiceSpecV2": { + "additionalProperties": false, + "properties": { + "ai": { + "discriminator": { + "mapping": { + "cascade": "#/$defs/CascadeAiConfig", + "realtime": "#/$defs/RealtimeAiConfig" + }, + "propertyName": "mode" + }, + "oneOf": [ + { + "$ref": "#/$defs/RealtimeAiConfig" + }, + { + "$ref": "#/$defs/CascadeAiConfig" + } + ], + "title": "Ai" + }, + "engine": { + "$ref": "#/$defs/EngineConfigV2" + }, + "media": { + "$ref": "#/$defs/MediaConfigV2" + }, + "observability": { + "$ref": "#/$defs/ObservabilityConfigV2" + }, + "transport": { + "$ref": "#/$defs/TransportConfig" + }, + "turn_taking": { + "$ref": "#/$defs/TurnTakingConfigV2" + } + }, + "required": [ + "engine", + "transport", + "media", + "turn_taking", + "ai" + ], + "title": "VoiceSpecV2", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "apiVersion": { + "const": "voxbench/v2", + "title": "Apiversion", + "type": "string" + }, + "kind": { + "const": "VoiceConfig", + "title": "Kind", + "type": "string" + }, + "meta": { + "$ref": "#/$defs/ConfigMeta" + }, + "spec": { + "$ref": "#/$defs/VoiceSpecV2" + } + }, + "required": [ + "apiVersion", + "kind", + "meta", + "spec" + ], + "title": "VoiceConfigV2", + "type": "object" +} diff --git a/schemas/manifest.v2.schema.json b/schemas/manifest.v2.schema.json new file mode 100644 index 0000000..c3e4335 --- /dev/null +++ b/schemas/manifest.v2.schema.json @@ -0,0 +1,437 @@ +{ + "$defs": { + "AudioFormat": { + "additionalProperties": false, + "properties": { + "channels": { + "exclusiveMinimum": 0, + "title": "Channels", + "type": "integer" + }, + "encoding": { + "enum": [ + "pcm16", + "pcm24", + "pcm32", + "float32" + ], + "title": "Encoding", + "type": "string" + }, + "rate": { + "exclusiveMinimum": 0, + "title": "Rate", + "type": "integer" + } + }, + "required": [ + "encoding", + "rate", + "channels" + ], + "title": "AudioFormat", + "type": "object" + }, + "AudioIoContract": { + "additionalProperties": false, + "properties": { + "accepts": { + "$ref": "#/$defs/AudioFormat" + }, + "mode": { + "enum": [ + "passthrough", + "rate_changing", + "format_changing" + ], + "title": "Mode", + "type": "string" + }, + "produces": { + "$ref": "#/$defs/AudioFormat" + } + }, + "required": [ + "mode", + "accepts", + "produces" + ], + "title": "AudioIoContract", + "type": "object" + }, + "CancellationCaps": { + "additionalProperties": false, + "properties": { + "conversation_truncation": { + "default": false, + "title": "Conversation Truncation", + "type": "boolean" + }, + "request": { + "default": false, + "title": "Request", + "type": "boolean" + }, + "terminal_ack": { + "default": false, + "title": "Terminal Ack", + "type": "boolean" + }, + "transport_abort": { + "default": false, + "title": "Transport Abort", + "type": "boolean" + } + }, + "title": "CancellationCaps", + "type": "object" + }, + "EngineCaps": { + "additionalProperties": false, + "properties": { + "authorities": { + "items": { + "enum": [ + "speech_detector", + "end_of_turn", + "interruption" + ], + "type": "string" + }, + "title": "Authorities", + "type": "array" + }, + "decoded_input": { + "$ref": "#/$defs/AudioFormat" + }, + "decoded_output": { + "$ref": "#/$defs/AudioFormat" + }, + "supported_codecs": { + "items": { + "minLength": 1, + "type": "string" + }, + "minItems": 1, + "title": "Supported Codecs", + "type": "array" + } + }, + "required": [ + "supported_codecs", + "decoded_input", + "decoded_output" + ], + "title": "EngineCaps", + "type": "object" + }, + "LifecycleCaps": { + "additionalProperties": false, + "properties": { + "final_output": { + "default": false, + "title": "Final Output", + "type": "boolean" + }, + "partial_output": { + "default": false, + "title": "Partial Output", + "type": "boolean" + }, + "response_boundaries": { + "default": false, + "title": "Response Boundaries", + "type": "boolean" + }, + "usage": { + "default": false, + "title": "Usage", + "type": "boolean" + } + }, + "title": "LifecycleCaps", + "type": "object" + }, + "OverridePermission": { + "additionalProperties": false, + "properties": { + "target": { + "title": "Target", + "type": "string" + }, + "type": { + "enum": [ + "string", + "integer", + "number", + "float", + "boolean", + "object", + "array" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "target", + "type" + ], + "title": "OverridePermission", + "type": "object" + }, + "ServiceCaps": { + "additionalProperties": false, + "properties": { + "audio_inputs": { + "items": { + "$ref": "#/$defs/AudioFormat" + }, + "title": "Audio Inputs", + "type": "array" + }, + "audio_outputs": { + "items": { + "$ref": "#/$defs/AudioFormat" + }, + "title": "Audio Outputs", + "type": "array" + }, + "authorities": { + "items": { + "enum": [ + "speech_detector", + "end_of_turn", + "interruption" + ], + "type": "string" + }, + "title": "Authorities", + "type": "array" + }, + "cancellation": { + "$ref": "#/$defs/CancellationCaps" + }, + "input_modality": { + "enum": [ + "audio", + "text" + ], + "title": "Input Modality", + "type": "string" + }, + "lifecycle": { + "$ref": "#/$defs/LifecycleCaps" + }, + "model_selection": { + "default": "explicit", + "enum": [ + "explicit", + "deployment_pinned" + ], + "title": "Model Selection", + "type": "string" + }, + "output_modality": { + "enum": [ + "audio", + "text" + ], + "title": "Output Modality", + "type": "string" + }, + "roles": { + "items": { + "enum": [ + "stt", + "llm", + "tts", + "realtime" + ], + "type": "string" + }, + "minItems": 1, + "title": "Roles", + "type": "array" + }, + "supported_models": { + "items": { + "minLength": 1, + "type": "string" + }, + "title": "Supported Models", + "type": "array" + } + }, + "required": [ + "roles", + "input_modality", + "output_modality" + ], + "title": "ServiceCaps", + "type": "object" + }, + "TextIoContract": { + "additionalProperties": false, + "properties": { + "input_modality": { + "const": "text", + "title": "Input Modality", + "type": "string" + }, + "output_modality": { + "const": "text", + "title": "Output Modality", + "type": "string" + } + }, + "required": [ + "input_modality", + "output_modality" + ], + "title": "TextIoContract", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "allowed_overrides": { + "items": { + "$ref": "#/$defs/OverridePermission" + }, + "title": "Allowed Overrides", + "type": "array" + }, + "apiVersion": { + "const": "voxbench/v2", + "title": "Apiversion", + "type": "string" + }, + "authorities": { + "items": { + "enum": [ + "speech_detector", + "end_of_turn", + "interruption" + ], + "type": "string" + }, + "title": "Authorities", + "type": "array" + }, + "engine_caps": { + "anyOf": [ + { + "$ref": "#/$defs/EngineCaps" + }, + { + "type": "null" + } + ], + "default": null + }, + "invariants_applicable": { + "items": { + "enum": [ + "duration_preserving", + "level_preserving", + "isochronous" + ], + "type": "string" + }, + "title": "Invariants Applicable", + "type": "array" + }, + "invariants_enforced": { + "items": { + "enum": [ + "duration_preserving", + "level_preserving", + "isochronous" + ], + "type": "string" + }, + "title": "Invariants Enforced", + "type": "array" + }, + "io": { + "anyOf": [ + { + "$ref": "#/$defs/AudioIoContract" + }, + { + "type": "null" + } + ], + "default": null + }, + "kind": { + "enum": [ + "engine", + "provider", + "processor" + ], + "title": "Kind", + "type": "string" + }, + "lossy_expected": { + "items": { + "type": "string" + }, + "title": "Lossy Expected", + "type": "array" + }, + "name": { + "minLength": 1, + "title": "Name", + "type": "string" + }, + "param_schema": { + "additionalProperties": true, + "title": "Param Schema", + "type": "object" + }, + "requires_host_capability": { + "items": { + "type": "string" + }, + "title": "Requires Host Capability", + "type": "array" + }, + "service_caps": { + "anyOf": [ + { + "$ref": "#/$defs/ServiceCaps" + }, + { + "type": "null" + } + ], + "default": null + }, + "text_io": { + "anyOf": [ + { + "$ref": "#/$defs/TextIoContract" + }, + { + "type": "null" + } + ], + "default": null + }, + "version": { + "minLength": 1, + "title": "Version", + "type": "string" + } + }, + "required": [ + "apiVersion", + "kind", + "name", + "version" + ], + "title": "CapabilityManifestV2", + "type": "object" +} diff --git a/schemas/service-event.v1.schema.json b/schemas/service-event.v1.schema.json new file mode 100644 index 0000000..a451ce2 --- /dev/null +++ b/schemas/service-event.v1.schema.json @@ -0,0 +1,258 @@ +{ + "additionalProperties": false, + "description": "One observed boundary with local aliases and no conversation content.", + "properties": { + "alignment_uncertainty_ms": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alignment Uncertainty Ms" + }, + "attributes": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "attempt_ordinal", + "sequence_ordinal", + "duration_ms", + "audio_duration_ms", + "input_units", + "output_units", + "cached_input_units", + "tool_call_count", + "queue_depth", + "sample_rate_hz", + "channels", + "encoding", + "retry", + "truncated", + "terminal_outcome", + "failure_alias", + "cancel_scope", + "observed_boundary", + "output_kind" + ] + }, + "title": "Attributes", + "type": "object" + }, + "clock_domain": { + "default": "application_wall", + "maxLength": 64, + "minLength": 1, + "title": "Clock Domain", + "type": "string" + }, + "collector_alias": { + "maxLength": 48, + "minLength": 1, + "title": "Collector Alias", + "type": "string" + }, + "component_id": { + "maxLength": 64, + "minLength": 1, + "title": "Component Id", + "type": "string" + }, + "event_alias": { + "maxLength": 48, + "minLength": 1, + "title": "Event Alias", + "type": "string" + }, + "generation_epoch": { + "default": 0, + "minimum": 0, + "title": "Generation Epoch", + "type": "integer" + }, + "kind": { + "enum": [ + "service.session_started", + "service.session_ended", + "stt.partial_emitted", + "stt.final_emitted", + "turn.speech_ended", + "turn.committed", + "llm.request_started", + "llm.first_output", + "llm.first_answer_text", + "llm.response_completed", + "aggregation.segment_ready", + "tts.request_started", + "tts.first_pcm", + "tts.response_completed", + "realtime.request_started", + "realtime.first_audio", + "realtime.response_completed", + "playback.write_started", + "playback.write_completed", + "playback.discarded", + "cancel.requested", + "cancel.acknowledged", + "service.failed" + ], + "title": "Kind", + "type": "string" + }, + "parent_request_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Request Alias" + }, + "request_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Request Alias" + }, + "response_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Response Alias" + }, + "role": { + "enum": [ + "stt", + "llm", + "tts", + "realtime", + "aggregation", + "coordinator", + "playback" + ], + "title": "Role", + "type": "string" + }, + "schema_version": { + "const": "voxbench/service-event/v1", + "default": "voxbench/service-event/v1", + "title": "Schema Version", + "type": "string" + }, + "segment_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Segment Alias" + }, + "session_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Session Alias" + }, + "ts": { + "format": "date-time", + "title": "Ts", + "type": "string" + }, + "turn_alias": { + "anyOf": [ + { + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Turn Alias" + }, + "unobserved_relations": { + "items": { + "enum": [ + "session", + "turn", + "request", + "parent_request", + "response", + "segment" + ], + "type": "string" + }, + "maxItems": 6, + "title": "Unobserved Relations", + "type": "array" + } + }, + "required": [ + "collector_alias", + "event_alias", + "kind", + "component_id", + "role" + ], + "title": "ServiceEvent", + "type": "object" +} diff --git a/src/voxbench/control_plane/run_api.py b/src/voxbench/control_plane/run_api.py index 8c1c6d8..1c9759e 100644 --- a/src/voxbench/control_plane/run_api.py +++ b/src/voxbench/control_plane/run_api.py @@ -92,7 +92,18 @@ RemoteRecordingUnavailableError, ) from voxbench.live_demo.simulated_bridge import run_simulated_live_bridge +from voxbench.observability.cascade_analysis import ( + CascadeTurnAnalysis, + analyze_cascade_service_events, +) +from voxbench.observability.service_events import SERVICE_EVENT_KINDS, ServiceEvent from voxbench.realtime_providers import GeminiLiveProvider, OpenAIRealtimeProvider +from voxbench.registry.config_views import ( + ai_mode, + pipeline_chains, + require_v1_workflow, + safe_ai_components, +) from voxbench.registry.errors import RegistryError from voxbench.registry.service import RegistryService from voxbench.verification import VerificationResult, verify_recordings @@ -328,6 +339,14 @@ class TimelineMetricPoint(BaseModel): value: float +class AiComponentResponse(BaseModel): + role: str + component_id: str + plugin: str + manifest_version: str | None = None + model: str | None = None + + class TimelineViolation(BaseModel): invariant: str passed: bool @@ -580,7 +599,19 @@ class TimelineEventObservationRequest(BaseModel): def validate_reference_fields(cls, value: str | None, info) -> str | None: if value is None: return None - return _validate_reference_text(value, info.field_name) + value = _validate_reference_text(value, info.field_name) + if info.field_name == "event_id" and value.startswith("service:"): + raise ValueError("service event IDs are reserved for service_events") + if info.field_name == "source" and value == "service_observation": + raise ValueError("service_observation source is reserved for service_events") + return value + + @field_validator("name") + @classmethod + def reject_reserved_service_name(cls, value: str) -> str: + if value in SERVICE_EVENT_KINDS: + raise ValueError("reserved service event names must use service_events") + return value @field_validator("attributes") @classmethod @@ -616,6 +647,7 @@ class ObservationBatchRequest(BaseModel): default_factory=list, max_length=128, ) + service_events: list[ServiceEvent] = Field(default_factory=list, max_length=128) @model_validator(mode="after") def require_observation(self) -> "ObservationBatchRequest": @@ -625,11 +657,17 @@ def require_observation(self) -> "ObservationBatchRequest": or self.sip_events or self.rtp_stats or self.timeline_events + or self.service_events ): raise ValueError("at least one observation is required") event_ids = [event.event_id for event in self.timeline_events] if len(event_ids) != len(set(event_ids)): raise ValueError("timeline event_id values must be unique within a batch") + if len(self.timeline_events) + len(self.service_events) > 128: + raise ValueError("timeline_events and service_events are limited to 128 combined") + service_ids = [event.normalized_event_id for event in self.service_events] + if len(service_ids) != len(set(service_ids)): + raise ValueError("service event collector/event aliases must be unique within a batch") return self @@ -640,6 +678,7 @@ class ObservationBatchResponse(BaseModel): sip_event_count: int rtp_stat_count: int timeline_event_count: int + service_event_count: int = 0 recording_count: int @@ -696,7 +735,7 @@ class TimelineLanes(BaseModel): sip_ladder: list[TimelineSipEvent] rtp_quality: list[TimelineRtpStat] stages: list[TimelineStageLane] - turns: list[dict[str, Any]] + turns: list[CascadeTurnAnalysis] host: list[TimelineMetricPoint] recordings: list[TimelineRecording] events: list[TimelineTypedEvent] @@ -710,6 +749,8 @@ class TimelineResponse(BaseModel): run_id: str t0: datetime config_hash: str + ai_mode: str + ai_components: list[AiComponentResponse] environment: RunEnvironmentMetadata readiness_checklist: list[ReadinessChecklistItem] readiness_summary: ReadinessSummaryResponse @@ -723,6 +764,8 @@ class RunResponse(BaseModel): conversation_id: str provider: str engine: str + ai_mode: str + ai_components: list[AiComponentResponse] status: str failure_alias: str | None recordings: list[RecordingResponse] @@ -739,6 +782,8 @@ class RunSummaryResponse(BaseModel): config_hash: str provider: str engine: str + ai_mode: str + ai_components: list[AiComponentResponse] status: str started_at: datetime ended_at: datetime | None @@ -797,6 +842,8 @@ class LiveRunStatusResponse(BaseModel): failure_alias: str | None started_at: datetime ended_at: datetime | None + ai_mode: str + ai_components: list[AiComponentResponse] environment_profile: EnvironmentProfile server_alias: str | None integration_target_alias: str | None @@ -842,6 +889,8 @@ def to_response(self) -> RunResponse: conversation_id=self.conversation_id, provider=self.provider, engine=self.engine, + ai_mode=_run_ai_mode(self.resolved_config), + ai_components=_run_ai_components(self.resolved_config), status=self.status, failure_alias=self.failure_alias, recordings=[RecordingResponse(**recording.__dict__) for recording in self.recordings], @@ -863,6 +912,8 @@ def to_summary(self) -> RunSummaryResponse: config_hash=self.config_hash, provider=self.provider, engine=self.engine, + ai_mode=_run_ai_mode(self.resolved_config), + ai_components=_run_ai_components(self.resolved_config), status=self.status, started_at=self.started_at, ended_at=self.ended_at, @@ -886,6 +937,8 @@ def to_live_status(self) -> LiveRunStatusResponse: failure_alias=self.failure_alias, started_at=self.started_at, ended_at=self.ended_at, + ai_mode=_run_ai_mode(self.resolved_config), + ai_components=_run_ai_components(self.resolved_config), environment_profile=self.environment.environment_profile, server_alias=self.environment.server_alias, integration_target_alias=self.environment.integration_target_alias, @@ -901,10 +954,7 @@ def to_live_status(self) -> LiveRunStatusResponse: ) def to_timeline(self) -> TimelineResponse: - stage_names = [ - stage["type"] - for stage in self.resolved_config["spec"]["media"]["pipeline"] - ] + stage_names = sorted(_configured_stage_names(self)) metrics_by_stage = { stage: [ TimelineMetricPoint( @@ -962,15 +1012,24 @@ def to_timeline(self) -> TimelineResponse: ) for stat in self.rtp_stats ] + cascade_turns = ( + analyze_cascade_service_events(self.timeline_events) + if _run_ai_mode(self.resolved_config) == "cascade" + else [] + ) typed_events = _typed_timeline_events(self) - typed_intervals = _typed_timeline_intervals(self) + typed_intervals = _typed_timeline_intervals(self, cascade_turns) typed_series = _typed_timeline_series(self) typed_artifacts = _typed_timeline_artifacts(self) typed_incidents = _typed_timeline_incidents(self, typed_artifacts) + typed_incidents.extend(_cascade_latency_incidents(self, cascade_turns)) + typed_incidents.sort(key=lambda item: (item.start_ms, item.incident_id)) return TimelineResponse( run_id=self.run_id, t0=self.started_at, config_hash=self.config_hash, + ai_mode=_run_ai_mode(self.resolved_config), + ai_components=_run_ai_components(self.resolved_config), environment=self.environment, readiness_checklist=self.readiness_checklist, readiness_summary=self.readiness_summary(), @@ -985,7 +1044,7 @@ def to_timeline(self) -> TimelineResponse: ) for stage in stage_names ], - turns=[], + turns=cascade_turns, host=host_metrics, recordings=[ TimelineRecording(**recording.__dict__) @@ -1874,9 +1933,7 @@ def _duration_contraction_evidence( def _pipeline_stage_names(run: StoredRun) -> list[str]: - return [ - stage["type"] for stage in run.resolved_config["spec"]["media"]["pipeline"] - ] + return sorted(_configured_stage_names(run)) def _plain_number(value: Any) -> float | None: @@ -2515,7 +2572,100 @@ def _typed_timeline_events(run: StoredRun) -> list[TimelineTypedEvent]: return sorted(events, key=lambda event: (event.t_rel_ms, event.event_id)) -def _typed_timeline_intervals(run: StoredRun) -> list[TimelineTypedInterval]: +def _service_latency_slos(run: StoredRun) -> dict[str, dict[str, Any]]: + observability = run.resolved_config.get("spec", {}).get("observability", {}) + contracts = observability.get("service_latency_slos", []) + if not isinstance(contracts, list): + return {} + return { + item["measurement"]: item + for item in contracts + if isinstance(item, dict) + and isinstance(item.get("measurement"), str) + and isinstance(item.get("id"), str) + and isinstance(item.get("max_ms"), int | float) + and not isinstance(item.get("max_ms"), bool) + } + + +def _cascade_latency_incidents( + run: StoredRun, + turns: list[CascadeTurnAnalysis], +) -> list[TimelineIncident]: + slos = _service_latency_slos(run) + if not slos: + return [] + event_by_id = {event.event_id: event for event in run.timeline_events} + incidents: list[TimelineIncident] = [] + for turn in turns: + for measurement in turn.measurements: + contract = slos.get(measurement.name) + if ( + contract is None + or measurement.status != "observed" + or measurement.duration_ms is None + or measurement.duration_ms <= contract["max_ms"] + or measurement.start_event_ref is None + or measurement.end_event_ref is None + ): + continue + start_event = event_by_id.get(measurement.start_event_ref) + end_event = event_by_id.get(measurement.end_event_ref) + if start_event is None or end_event is None: + continue + incidents.append( + TimelineIncident( + incident_id=f"cascade-latency:{contract['id']}:{measurement.measurement_id}", + rule_id="cascade_latency_slo_v1", + category=( + "conversation" + if measurement.name + in { + "stt_finalization_wait", + "turn_coordination_wait", + "end_to_end_local_response_wait", + } + else "provider" + ), + severity="warning", + title=f"Cascade latency SLO exceeded: {measurement.name}", + summary=( + f"{measurement.duration_ms:g} ms exceeded configured " + f"{contract['max_ms']:g} ms" + ), + start_ms=_relative_seconds(start_event.ts, run.started_at) * 1000, + end_ms=_relative_seconds(end_event.ts, run.started_at) * 1000, + confidence="high", + stage=( + measurement.component_ids[-1] + if measurement.component_ids + else None + ), + observed={ + "duration_ms": measurement.duration_ms, + "definition": measurement.definition, + "definition_version": measurement.definition_version, + "clock_domain": measurement.clock_domain, + "alignment_uncertainty_ms": ( + measurement.alignment_uncertainty_ms + ), + "scope": measurement.scope.model_dump(exclude_none=True), + "remote_playout_observed": False, + }, + expected={ + "service_latency_slo_id": contract["id"], + "duration_ms_at_or_below": contract["max_ms"], + }, + evidence_refs=measurement.evidence_refs, + ) + ) + return incidents + + +def _typed_timeline_intervals( + run: StoredRun, + cascade_turns: list[CascadeTurnAnalysis] | None = None, +) -> list[TimelineTypedInterval]: intervals: list[TimelineTypedInterval] = [] origin_ns = int(run.started_at.timestamp() * 1_000_000_000) for index, span in enumerate(run.spans): @@ -2536,6 +2686,69 @@ def _typed_timeline_intervals(run: StoredRun) -> list[TimelineTypedInterval]: attributes={"duration_ms": end_ms - start_ms}, ) ) + event_by_id = {event.event_id: event for event in run.timeline_events} + latency_slos = _service_latency_slos(run) + cascade_categories: dict[str, TimelineCategory] = { + "stt_finalization_wait": "conversation", + "turn_coordination_wait": "conversation", + "llm_dispatch_wait": "provider", + "llm_first_output_wait": "provider", + "llm_first_answer_text_wait": "provider", + "text_aggregation_wait": "pipeline", + "tts_dispatch_queue_wait": "provider", + "tts_first_audio_wait": "provider", + "output_playback_start_wait": "buffer", + "end_to_end_local_response_wait": "conversation", + } + for turn in cascade_turns or []: + for measurement in turn.measurements: + if ( + measurement.status != "observed" + or measurement.start_event_ref is None + or measurement.end_event_ref is None + ): + continue + start_event = event_by_id.get(measurement.start_event_ref) + end_event = event_by_id.get(measurement.end_event_ref) + if start_event is None or end_event is None: + continue + intervals.append( + TimelineTypedInterval( + interval_id=measurement.measurement_id, + category=cascade_categories[measurement.name], + name=f"cascade.{measurement.name}", + start_ms=_relative_seconds(start_event.ts, run.started_at) * 1000, + end_ms=_relative_seconds(end_event.ts, run.started_at) * 1000, + clock_domain=measurement.clock_domain or start_event.clock_domain, + alignment_uncertainty_ms=measurement.alignment_uncertainty_ms, + stage=(measurement.component_ids[-1] if measurement.component_ids else None), + source="cascade_causal_analysis_v1", + correlation_alias=turn.turn_alias, + attributes={ + "duration_ms": measurement.duration_ms, + "definition": measurement.definition, + "definition_version": measurement.definition_version, + "evidence_refs": measurement.evidence_refs, + "scope": measurement.scope.model_dump(exclude_none=True), + "remote_playout_observed": False, + "latency_threshold_status": ( + "not_configured" + if measurement.name not in latency_slos + else "exceeded" + if measurement.duration_ms is not None + and measurement.duration_ms + > latency_slos[measurement.name]["max_ms"] + else "within" + ), + "latency_slo_id": ( + latency_slos.get(measurement.name, {}).get("id") + ), + "latency_slo_max_ms": ( + latency_slos.get(measurement.name, {}).get("max_ms") + ), + }, + ) + ) for item in _rtp_packet_discontinuity_evidence(run): intervals.append( TimelineTypedInterval( @@ -4094,26 +4307,39 @@ def get_run_api_state(request: Request) -> RunApiState: RunApiStateDependency = Annotated[RunApiState, Depends(get_run_api_state)] -def _resolve_run_request(request: RunCreateRequest): +def _resolve_run_request(request: RunCreateRequest, *, allow_v2_observation: bool = False): registry = RegistryService() try: for manifest in request.manifests: registry.register_manifest(manifest) for config in request.configs: registry.register_config(config) - return registry.resolve_config(request.config_name) + resolved = registry.resolve_config(request.config_name) + if not allow_v2_observation: + require_v1_workflow(resolved.resolved, operation="run creation/execution") + return resolved except RegistryError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc def _create_running_run(request: RunCreateRequest, resolved: Any) -> StoredRun: run_id = str(uuid4()) + mode = ai_mode(resolved.resolved) + provider = ( + "cascade" + if mode == "cascade" + else ( + resolved.resolved["spec"]["ai"]["provider"] + if resolved.resolved.get("apiVersion", "voxbench/v1") == "voxbench/v1" + else resolved.resolved["spec"]["ai"]["realtime"]["plugin"] + ) + ) return StoredRun( run_id=run_id, config_hash=resolved.hash, call_id=request.call_id, conversation_id="", - provider=resolved.resolved["spec"]["ai"]["provider"], + provider=provider, engine=resolved.resolved["spec"]["engine"]["kind"], status="running", started_at=datetime.now(UTC), @@ -4371,11 +4597,169 @@ def _provider_readiness(request: LiveDemoRunRequest): def _configured_stage_names(stored: StoredRun) -> set[str]: + chains = pipeline_chains(stored.resolved_config) + legacy = stored.resolved_config.get("apiVersion", "voxbench/v1") == "voxbench/v1" + identity = "type" if legacy else "id" return { - stage["type"] - for stage in stored.resolved_config["spec"]["media"]["pipeline"] - if isinstance(stage, dict) and isinstance(stage.get("type"), str) + stage[identity] + for stages in chains.values() + for stage in stages + if isinstance(stage, dict) and isinstance(stage.get(identity), str) + } + + +def _run_ai_mode(config: dict[str, Any]) -> str: + ai = config.get("spec", {}).get("ai") + return ai_mode(config) if isinstance(ai, dict) else "realtime" + + +def _run_ai_components(config: dict[str, Any]) -> list[dict[str, str]]: + ai = config.get("spec", {}).get("ai") + return safe_ai_components(config) if isinstance(ai, dict) else [] + + +MAX_TIMELINE_EVENTS_PER_RUN = 10_000 + + +def _service_component_roles(config: dict[str, Any]) -> dict[str, set[str]]: + if config.get("apiVersion", "voxbench/v1") == "voxbench/v1": + return {"ai": {"realtime"}, "application": {"coordinator"}} + ai = config["spec"]["ai"] + roles: dict[str, set[str]] = {"application": {"coordinator"}} + if ai["mode"] == "realtime": + roles[ai["realtime"]["id"]] = {"realtime"} + else: + for role in ("stt", "llm", "tts"): + roles[ai[role]["id"]] = {role} + roles[ai["text_aggregation"]["id"]] = {"aggregation"} + roles.setdefault("engine", set()).add("coordinator") + for stage in config["spec"]["media"]["output_pipeline"]: + roles.setdefault(stage["id"], set()).add("playback") + return roles + + +def _validate_service_component(stored: StoredRun, event: ServiceEvent) -> None: + roles = _service_component_roles(stored.resolved_config) + if event.role not in roles.get(event.component_id, set()): + raise HTTPException( + status_code=400, + detail=( + f"component '{event.component_id}' is not configured for service role " + f"'{event.role}'" + ), + ) + authority_name = ( + "speech_detector" + if event.kind == "turn.speech_ended" + else "end_of_turn" + if event.kind == "turn.committed" + else None + ) + if authority_name is not None: + authority = stored.resolved_config["spec"]["turn_taking"].get(authority_name) + if authority is not None and event.component_id != authority: + raise HTTPException( + status_code=400, + detail=( + f"component '{event.component_id}' is not the configured " + f"{authority_name} authority" + ), + ) + + +def _service_event_artifact(event: ServiceEvent) -> TimelineEventArtifact: + relation_attributes = { + name: value + for name, value in { + "session_alias": event.session_alias, + "turn_alias": event.turn_alias, + "request_alias": event.request_alias, + "parent_request_alias": event.parent_request_alias, + "response_alias": event.response_alias, + "segment_alias": event.segment_alias, + }.items() + if value is not None } + attributes: dict[str, Any] = { + "service_role": event.role, + "generation_epoch": event.generation_epoch, + **relation_attributes, + **event.attributes, + } + if event.unobserved_relations: + attributes["unobserved_relations"] = ",".join(event.unobserved_relations) + category = ( + "pipeline" + if event.role == "aggregation" + else "buffer" + if event.role == "playback" + else "conversation" + if event.role == "coordinator" + else "provider" + ) + correlation = ( + event.request_alias + or event.turn_alias + or event.response_alias + or event.segment_alias + or event.session_alias + ) + return TimelineEventArtifact( + event_id=event.normalized_event_id, + category=category, + name=event.kind, + ts=event.ts, + clock_domain=event.clock_domain, + alignment_uncertainty_ms=event.alignment_uncertainty_ms, + stage=event.component_id, + source="service_observation", + correlation_alias=correlation, + attributes=attributes, + ) + + +def _without_derived_service_coverage(event: TimelineEventArtifact) -> TimelineEventArtifact: + attributes = dict(event.attributes) + attributes.pop("unresolved_relations", None) + return TimelineEventArtifact(**{**event.__dict__, "attributes": attributes}) + + +def _refresh_service_coverage(events: list[TimelineEventArtifact]) -> None: + known_requests = { + event.attributes["request_alias"] + for event in events + if event.source == "service_observation" + and isinstance(event.attributes.get("request_alias"), str) + } + for index, event in enumerate(events): + if event.source != "service_observation": + continue + attributes = dict(event.attributes) + parent = attributes.get("parent_request_alias") + if isinstance(parent, str) and parent not in known_requests: + attributes["unresolved_relations"] = "parent_request" + else: + attributes.pop("unresolved_relations", None) + if len(attributes) > 16: + raise HTTPException(status_code=400, detail="normalized service attributes exceed 16") + events[index] = TimelineEventArtifact(**{**event.__dict__, "attributes": attributes}) + + +def _timeline_event_artifact(event: TimelineEventObservationRequest) -> TimelineEventArtifact: + return TimelineEventArtifact( + event_id=event.event_id, + category=event.category, + name=event.name, + ts=event.ts, + clock_domain=event.clock_domain, + alignment_uncertainty_ms=event.alignment_uncertainty_ms, + direction=event.direction, + stage=event.stage, + stream_alias=event.stream_alias, + source=event.source, + correlation_alias=event.correlation_alias, + attributes=event.attributes, + ) def _decode_audio_chunk(chunk: AudioChunkObservationRequest) -> bytes: @@ -4600,7 +4984,7 @@ async def create_observed_run( request: RunCreateRequest, api_state: RunApiStateDependency, ) -> RunResponse: - resolved = _resolve_run_request(request) + resolved = _resolve_run_request(request, allow_v2_observation=True) stored = _create_running_run(request, resolved) stored.conversation_id = f"observed-{stored.run_id}" api_state.repository.save(stored) @@ -4734,6 +5118,51 @@ async def ingest_observation_batch( detail=f"run '{request.run_id}' is not accepting observations", ) + if ( + stored.resolved_config.get("apiVersion", "voxbench/v1") == "voxbench/v2" + and request.audio_chunks + ): + raise HTTPException( + status_code=400, + detail="v2 service observation does not support audio chunks until recording maps", + ) + + for event in request.service_events: + _validate_service_component(stored, event) + + existing_by_id = {event.event_id: event for event in stored.timeline_events} + generic_candidates = [_timeline_event_artifact(event) for event in request.timeline_events] + service_candidates = [_service_event_artifact(event) for event in request.service_events] + new_events: list[TimelineEventArtifact] = [] + for candidate in generic_candidates: + previous = existing_by_id.get(candidate.event_id) + if previous is not None and previous != candidate: + raise HTTPException( + status_code=409, + detail=f"timeline event '{candidate.event_id}' conflicts with stored payload", + ) + if previous is None: + new_events.append(candidate) + existing_by_id[candidate.event_id] = candidate + for candidate in service_candidates: + previous = existing_by_id.get(candidate.event_id) + if previous is not None and ( + _without_derived_service_coverage(previous) + != _without_derived_service_coverage(candidate) + ): + raise HTTPException( + status_code=409, + detail=f"service event '{candidate.event_id}' conflicts with stored payload", + ) + if previous is None: + new_events.append(candidate) + existing_by_id[candidate.event_id] = candidate + if len(stored.timeline_events) + len(new_events) > MAX_TIMELINE_EVENTS_PER_RUN: + raise HTTPException( + status_code=429, + detail=f"run observation limit of {MAX_TIMELINE_EVENTS_PER_RUN} events exceeded", + ) + configured_stages = _configured_stage_names(stored) referenced_stages = { metric.stage for metric in request.metrics if metric.stage is not None @@ -4808,25 +5237,8 @@ async def ingest_observation_batch( ) for stat in request.rtp_stats ) - existing_event_ids = {event.event_id for event in stored.timeline_events} - stored.timeline_events.extend( - TimelineEventArtifact( - event_id=event.event_id, - category=event.category, - name=event.name, - ts=event.ts, - clock_domain=event.clock_domain, - alignment_uncertainty_ms=event.alignment_uncertainty_ms, - direction=event.direction, - stage=event.stage, - stream_alias=event.stream_alias, - source=event.source, - correlation_alias=event.correlation_alias, - attributes=event.attributes, - ) - for event in request.timeline_events - if event.event_id not in existing_event_ids - ) + stored.timeline_events.extend(new_events) + _refresh_service_coverage(stored.timeline_events) api_state.repository.save(stored) return ObservationBatchResponse( run_id=stored.run_id, @@ -4835,6 +5247,7 @@ async def ingest_observation_batch( sip_event_count=len(request.sip_events), rtp_stat_count=len(request.rtp_stats), timeline_event_count=len(request.timeline_events), + service_event_count=len(request.service_events), recording_count=len(stored.recordings), ) @@ -4851,10 +5264,14 @@ async def complete_observed_run( if stored.status != "running": raise HTTPException(status_code=409, detail=f"run '{run_id}' cannot be completed") - stored.verifications = verify_recordings( - resolved_config=stored.resolved_config, - recordings=stored.recordings, - metrics=stored.metrics, + stored.verifications = ( + verify_recordings( + resolved_config=stored.resolved_config, + recordings=stored.recordings, + metrics=stored.metrics, + ) + if stored.resolved_config.get("apiVersion", "voxbench/v1") == "voxbench/v1" + else [] ) stored.status = "completed" stored.ended_at = datetime.now(UTC) diff --git a/src/voxbench/engine_harness/harness.py b/src/voxbench/engine_harness/harness.py index 1278245..9dd5fca 100644 --- a/src/voxbench/engine_harness/harness.py +++ b/src/voxbench/engine_harness/harness.py @@ -12,6 +12,7 @@ from voxbench.engine_harness.plan import build_stage_plan from voxbench.engine_harness.storage import RecordingSink from voxbench.engine_harness.telemetry import HarnessTracer, span_attrs +from voxbench.registry.config_views import require_v1_workflow class EngineHarness: @@ -33,6 +34,7 @@ def run_once( resolved_config: dict[str, object], config_hash: str, ) -> HarnessResult: + require_v1_workflow(resolved_config, operation="legacy harness execution") conversation_id = str(uuid4()) recordings = [] metrics = [] diff --git a/src/voxbench/engine_harness/plan.py b/src/voxbench/engine_harness/plan.py index e78e1d1..9e74d03 100644 --- a/src/voxbench/engine_harness/plan.py +++ b/src/voxbench/engine_harness/plan.py @@ -5,9 +5,11 @@ from typing import Any from voxbench.engine_harness.models import StagePlan +from voxbench.registry.config_views import require_v1_workflow def build_stage_plan(resolved_config: dict[str, Any]) -> list[StagePlan]: + require_v1_workflow(resolved_config, operation="legacy harness stage planning") stages = resolved_config["spec"]["media"]["pipeline"] return [ StagePlan( @@ -30,4 +32,3 @@ def _stage_format(stage: dict[str, Any]) -> dict[str, Any]: for key in ("encoding", "rate", "channels", "output_rate") if key in params } - diff --git a/src/voxbench/observability/__init__.py b/src/voxbench/observability/__init__.py index 229d6d2..19ff8b7 100644 --- a/src/voxbench/observability/__init__.py +++ b/src/voxbench/observability/__init__.py @@ -18,8 +18,15 @@ detect_pcm_s16le_discontinuity, rtp_packet_from_datagram, ) +from voxbench.observability.service_events import ( + SERVICE_EVENT_KINDS, + ServiceEvent, + ServiceEventKind, + ServiceEventRole, +) __all__ = [ + "SERVICE_EVENT_KINDS", "AudioChunk", "HttpObservationTransport", "MetricPoint", @@ -30,6 +37,9 @@ "RtpPacket", "RtpPacketTapAdapter", "RtpStats", + "ServiceEvent", + "ServiceEventKind", + "ServiceEventRole", "SipEvent", "TimelineCategory", "TimelineEvent", diff --git a/src/voxbench/observability/cascade_analysis.py b/src/voxbench/observability/cascade_analysis.py new file mode 100644 index 0000000..cfb4e59 --- /dev/null +++ b/src/voxbench/observability/cascade_analysis.py @@ -0,0 +1,641 @@ +"""Deterministic causal timing derived from metadata-only service events.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from voxbench.engine_harness.models import TimelineEventArtifact + +AnalysisStatus = Literal["observed", "unobserved", "indeterminate"] +CriticalPathStatus = Literal["complete", "incomplete", "indeterminate"] +MeasurementName = Literal[ + "stt_finalization_wait", + "turn_coordination_wait", + "llm_dispatch_wait", + "llm_first_output_wait", + "llm_first_answer_text_wait", + "text_aggregation_wait", + "tts_dispatch_queue_wait", + "tts_first_audio_wait", + "output_playback_start_wait", + "end_to_end_local_response_wait", +] + +ANALYSIS_SCHEMA_VERSION = "voxbench/cascade-analysis/v1" +MEASUREMENT_DEFINITION_VERSION = "cascade-latency/v1" + + +class CausalScope(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + turn_alias: str + generation_epoch: int + operation_request_alias: str | None = None + response_alias: str | None = None + segment_alias: str | None = None + request_alias: str | None = None + + +class ServiceLatencyMeasurement(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + measurement_id: str + name: MeasurementName + status: AnalysisStatus + definition_version: Literal["cascade-latency/v1"] = MEASUREMENT_DEFINITION_VERSION + definition: str + scope: CausalScope + duration_ms: float | None = None + start_event_ref: str | None = None + end_event_ref: str | None = None + evidence_refs: list[str] = Field(default_factory=list) + component_ids: list[str] = Field(default_factory=list) + clock_domain: str | None = None + alignment_uncertainty_ms: float | None = None + reason_alias: str | None = None + + +class CascadeCriticalPath(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + status: CriticalPathStatus + measurement_ids: list[str] + end_to_end_measurement_id: str + covered_duration_ms: float + unexplained_duration_ms: float | None + reason_alias: str | None = None + + +class CascadeCoverage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + missing_boundaries: list[str] + explicit_unobserved_relations: list[str] + unresolved_relations: list[str] + remote_playout_observed: Literal[False] = False + + +class CascadeTurnAnalysis(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["voxbench/cascade-analysis/v1"] = ANALYSIS_SCHEMA_VERSION + turn_alias: str + response_alias: str | None + operation_request_alias: str | None + generation_epoch: int + completion_observed: bool + measurements: list[ServiceLatencyMeasurement] + critical_path: CascadeCriticalPath + coverage: CascadeCoverage + + +_DEFINITIONS: dict[MeasurementName, str] = { + "stt_finalization_wait": ( + "independent caller speech end to the correlated STT final boundary" + ), + "turn_coordination_wait": ( + "correlated STT final boundary to committed conversational turn" + ), + "llm_dispatch_wait": "committed turn to the correlated LLM request start", + "llm_first_output_wait": ( + "LLM request start to first observed output of any declared output kind" + ), + "llm_first_answer_text_wait": ( + "LLM request start to first answer text eligible for speech" + ), + "text_aggregation_wait": ( + "first answer text to a correlated segment becoming ready for TTS" + ), + "tts_dispatch_queue_wait": ( + "correlated segment-ready boundary to the actual TTS request start" + ), + "tts_first_audio_wait": "TTS request start to first correlated PCM output", + "output_playback_start_wait": ( + "first correlated TTS PCM output to first matching local frame write" + ), + "end_to_end_local_response_wait": ( + "independent caller speech end to first causally matching local frame write" + ), +} + + +def analyze_cascade_service_events( + events: Iterable[TimelineEventArtifact], +) -> list[CascadeTurnAnalysis]: + """Group explicit aliases and derive timings without nearest-time matching.""" + + service_events = sorted( + (event for event in events if event.source == "service_observation"), + key=lambda event: (event.ts, event.event_id), + ) + llm_starts = [event for event in service_events if event.name == "llm.request_started"] + analyses = [ + _analyze_llm_operation(service_events, llm_start) + for llm_start in llm_starts + if _string_attr(llm_start, "turn_alias") is not None + ] + return sorted( + analyses, + key=lambda item: ( + item.turn_alias, + item.generation_epoch, + item.response_alias or "", + item.operation_request_alias or "", + ), + ) + + +def _analyze_llm_operation( + events: list[TimelineEventArtifact], + llm_start: TimelineEventArtifact, +) -> CascadeTurnAnalysis: + turn = _required_attr(llm_start, "turn_alias") + llm_request = _required_attr(llm_start, "request_alias") + response = _string_attr(llm_start, "response_alias") + epoch = _epoch(llm_start) + request_lineage = _request_lineage(events, llm_start, epoch=epoch) + stt_request = next( + ( + request + for request in reversed(request_lineage) + if _first( + events, + "stt.final_emitted", + request_alias=request, + generation_epoch=epoch, + ) + is not None + ), + _string_attr(llm_start, "parent_request_alias"), + ) + base_scope = CausalScope( + turn_alias=turn, + generation_epoch=epoch, + operation_request_alias=llm_request, + response_alias=response, + ) + + turn_commit = _first( + events, + "turn.committed", + turn_alias=turn, + parent_request_alias=stt_request, + generation_epoch=epoch, + ) + stt_final = _first( + events, + "stt.final_emitted", + request_alias=stt_request, + generation_epoch=epoch, + ) + speech_end = _first( + events, + "turn.speech_ended", + turn_alias=turn, + request_alias=stt_request, + generation_epoch=epoch, + ) + first_output = _first( + events, + "llm.first_output", + request_alias=llm_request, + response_alias=response, + generation_epoch=epoch, + ) + first_answer = _first( + events, + "llm.first_answer_text", + request_alias=llm_request, + response_alias=response, + generation_epoch=epoch, + ) + llm_complete = _first( + events, + "llm.response_completed", + request_alias=llm_request, + response_alias=response, + generation_epoch=epoch, + ) + + measurements = [ + _measurement("stt_finalization_wait", speech_end, stt_final, base_scope, "stt"), + _measurement("turn_coordination_wait", stt_final, turn_commit, base_scope, "turn"), + _measurement("llm_dispatch_wait", turn_commit, llm_start, base_scope, "llm"), + _measurement("llm_first_output_wait", llm_start, first_output, base_scope, "llm"), + _measurement( + "llm_first_answer_text_wait", llm_start, first_answer, base_scope, "llm" + ), + ] + + aggregations = _matching( + events, + "aggregation.segment_ready", + parent_request_alias=llm_request, + response_alias=response, + generation_epoch=epoch, + ) + path_candidates: list[tuple[TimelineEventArtifact, list[str]]] = [] + tts_starts: list[TimelineEventArtifact] = [] + playback_starts: list[TimelineEventArtifact] = [] + if not aggregations: + measurements.append( + _measurement( + "text_aggregation_wait", first_answer, None, base_scope, "aggregation" + ) + ) + + for aggregation in aggregations: + segment = _string_attr(aggregation, "segment_alias") + aggregation_request = _string_attr(aggregation, "request_alias") + segment_scope = base_scope.model_copy( + update={"segment_alias": segment, "request_alias": aggregation_request} + ) + aggregation_measurement = _measurement( + "text_aggregation_wait", + first_answer, + aggregation, + segment_scope, + "aggregation", + ) + measurements.append(aggregation_measurement) + segment_tts_starts = _matching( + events, + "tts.request_started", + parent_request_alias=aggregation_request, + response_alias=response, + segment_alias=segment, + generation_epoch=epoch, + ) + if not segment_tts_starts: + measurements.append( + _measurement( + "tts_dispatch_queue_wait", + aggregation, + None, + segment_scope, + "tts", + ) + ) + for tts_start in segment_tts_starts: + tts_starts.append(tts_start) + tts_request = _string_attr(tts_start, "request_alias") + tts_scope = segment_scope.model_copy(update={"request_alias": tts_request}) + dispatch = _measurement( + "tts_dispatch_queue_wait", aggregation, tts_start, tts_scope, "tts" + ) + first_pcm = _first( + events, + "tts.first_pcm", + request_alias=tts_request, + response_alias=response, + segment_alias=segment, + generation_epoch=epoch, + ) + first_audio = _measurement( + "tts_first_audio_wait", tts_start, first_pcm, tts_scope, "tts" + ) + playback = _first( + events, + "playback.write_started", + parent_request_alias=tts_request, + response_alias=response, + segment_alias=segment, + generation_epoch=epoch, + ) + if playback is not None: + playback_starts.append(playback) + playback_wait = _measurement( + "output_playback_start_wait", first_pcm, playback, tts_scope, "playback" + ) + measurements.extend((dispatch, first_audio, playback_wait)) + if playback is not None: + path_candidates.append( + ( + playback, + [ + measurements[0].measurement_id, + measurements[1].measurement_id, + measurements[2].measurement_id, + measurements[4].measurement_id, + aggregation_measurement.measurement_id, + dispatch.measurement_id, + first_audio.measurement_id, + playback_wait.measurement_id, + ], + ) + ) + + selected_path = ( + min(path_candidates, key=lambda item: (item[0].ts, item[0].event_id)) + if path_candidates + else None + ) + selected_playback = selected_path[0] if selected_path is not None else None + end_to_end = _measurement( + "end_to_end_local_response_wait", + speech_end, + selected_playback, + base_scope, + "response", + reason_alias="causal_path_incomplete" if selected_playback is None else None, + ) + measurements.append(end_to_end) + + path_ids = selected_path[1] if selected_path is not None else [ + measurements[0].measurement_id, + measurements[1].measurement_id, + measurements[2].measurement_id, + measurements[4].measurement_id, + ] + path_measurements = [ + measurement for measurement in measurements if measurement.measurement_id in path_ids + ] + critical_path = _critical_path(path_measurements, path_ids, end_to_end) + + relevant = _relevant_events( + events, + turn=turn, + response=response, + request_aliases={ + alias + for alias in ( + stt_request, + llm_request, + *request_lineage, + *(_string_attr(item, "request_alias") for item in aggregations), + *(_string_attr(item, "request_alias") for item in tts_starts), + ) + if alias is not None + }, + epoch=epoch, + ) + missing_boundaries = sorted( + { + measurement.name + for measurement in measurements + if measurement.status != "observed" + } + ) + explicit_unobserved = sorted( + f"{event.event_id}:{relation}" + for event in relevant + for relation in _csv_attr(event, "unobserved_relations") + ) + unresolved = sorted( + f"{event.event_id}:{relation}" + for event in relevant + for relation in _csv_attr(event, "unresolved_relations") + ) + tts_terminal_requests = { + _string_attr(event, "request_alias") + for event in relevant + if event.name == "tts.response_completed" + } + playback_terminal_requests = { + _string_attr(event, "parent_request_alias") + for event in relevant + if event.name in {"playback.write_completed", "playback.discarded"} + } + started_tts_requests = {_string_attr(event, "request_alias") for event in tts_starts} + started_playback_requests = { + _string_attr(event, "parent_request_alias") for event in playback_starts + } + completion_observed = bool(aggregations) and llm_complete is not None and ( + started_tts_requests <= tts_terminal_requests + and started_playback_requests <= playback_terminal_requests + ) + return CascadeTurnAnalysis( + turn_alias=turn, + response_alias=response, + operation_request_alias=llm_request, + generation_epoch=epoch, + completion_observed=completion_observed, + measurements=measurements, + critical_path=critical_path, + coverage=CascadeCoverage( + missing_boundaries=missing_boundaries, + explicit_unobserved_relations=explicit_unobserved, + unresolved_relations=unresolved, + ), + ) + + +def _measurement( + name: MeasurementName, + start: TimelineEventArtifact | None, + end: TimelineEventArtifact | None, + scope: CausalScope, + identity_suffix: str, + *, + reason_alias: str | None = None, +) -> ServiceLatencyMeasurement: + request = scope.request_alias or scope.operation_request_alias or "unknown" + segment = scope.segment_alias or "none" + measurement_id = ( + f"cascade:{scope.turn_alias}:{scope.generation_epoch}:{request}:{segment}:" + f"{identity_suffix}:{name}" + ) + evidence = [event.event_id for event in (start, end) if event is not None] + components = sorted({event.stage for event in (start, end) if event and event.stage}) + if start is None or end is None: + return ServiceLatencyMeasurement( + measurement_id=measurement_id, + name=name, + status="unobserved", + definition=_DEFINITIONS[name], + scope=scope, + start_event_ref=start.event_id if start else None, + end_event_ref=end.event_id if end else None, + evidence_refs=evidence, + component_ids=components, + reason_alias=reason_alias or "boundary_unobserved", + ) + if start.clock_domain != end.clock_domain: + return ServiceLatencyMeasurement( + measurement_id=measurement_id, + name=name, + status="indeterminate", + definition=_DEFINITIONS[name], + scope=scope, + start_event_ref=start.event_id, + end_event_ref=end.event_id, + evidence_refs=evidence, + component_ids=components, + reason_alias="clock_domain_mismatch", + ) + duration_ms = (end.ts - start.ts).total_seconds() * 1000 + if duration_ms < 0: + return ServiceLatencyMeasurement( + measurement_id=measurement_id, + name=name, + status="indeterminate", + definition=_DEFINITIONS[name], + scope=scope, + start_event_ref=start.event_id, + end_event_ref=end.event_id, + evidence_refs=evidence, + component_ids=components, + clock_domain=start.clock_domain, + reason_alias="end_precedes_start", + ) + uncertainty = ( + start.alignment_uncertainty_ms + end.alignment_uncertainty_ms + if start.alignment_uncertainty_ms is not None + and end.alignment_uncertainty_ms is not None + else None + ) + return ServiceLatencyMeasurement( + measurement_id=measurement_id, + name=name, + status="observed", + definition=_DEFINITIONS[name], + scope=scope, + duration_ms=duration_ms, + start_event_ref=start.event_id, + end_event_ref=end.event_id, + evidence_refs=evidence, + component_ids=components, + clock_domain=start.clock_domain, + alignment_uncertainty_ms=uncertainty, + ) + + +def _critical_path( + path_measurements: list[ServiceLatencyMeasurement], + path_ids: list[str], + end_to_end: ServiceLatencyMeasurement, +) -> CascadeCriticalPath: + ordered = {measurement.measurement_id: measurement for measurement in path_measurements} + values = [ordered[item] for item in path_ids if item in ordered] + if ( + any(item.status == "indeterminate" for item in values) + or end_to_end.status == "indeterminate" + ): + status: CriticalPathStatus = "indeterminate" + reason = "clock_or_order_indeterminate" + elif ( + len(values) != len(path_ids) + or any(item.status != "observed" for item in values) + or end_to_end.status != "observed" + ): + status = "incomplete" + reason = "causal_boundary_unobserved" + else: + status = "complete" + reason = None + covered = sum(item.duration_ms or 0.0 for item in values if item.status == "observed") + unexplained = ( + max(0.0, (end_to_end.duration_ms or 0.0) - covered) + if end_to_end.status == "observed" + else None + ) + return CascadeCriticalPath( + status=status, + measurement_ids=path_ids, + end_to_end_measurement_id=end_to_end.measurement_id, + covered_duration_ms=covered, + unexplained_duration_ms=unexplained, + reason_alias=reason, + ) + + +def _matching( + events: list[TimelineEventArtifact], + name: str, + **attributes: str | int | None, +) -> list[TimelineEventArtifact]: + return [ + event + for event in events + if event.name == name + and all( + (_epoch(event) if key == "generation_epoch" else _string_attr(event, key)) + == value + for key, value in attributes.items() + ) + ] + + +def _first( + events: list[TimelineEventArtifact], + name: str, + **attributes: str | int | None, +) -> TimelineEventArtifact | None: + matches = _matching(events, name, **attributes) + return min(matches, key=lambda event: (event.ts, event.event_id)) if matches else None + + +def _relevant_events( + events: list[TimelineEventArtifact], + *, + turn: str, + response: str | None, + request_aliases: set[str], + epoch: int, +) -> list[TimelineEventArtifact]: + return [ + event + for event in events + if _epoch(event) == epoch + and ( + _string_attr(event, "turn_alias") == turn + or (response is not None and _string_attr(event, "response_alias") == response) + or _string_attr(event, "request_alias") in request_aliases + or _string_attr(event, "parent_request_alias") in request_aliases + ) + ] + + +def _request_lineage( + events: list[TimelineEventArtifact], + event: TimelineEventArtifact, + *, + epoch: int, +) -> list[str]: + """Walk explicit request parents without inferring links from timestamp proximity.""" + + lineage: list[str] = [] + current = _string_attr(event, "parent_request_alias") + while current is not None and current not in lineage: + lineage.append(current) + parent_event = next( + ( + candidate + for candidate in events + if _string_attr(candidate, "request_alias") == current + and _string_attr(candidate, "parent_request_alias") is not None + and _epoch(candidate) == epoch + ), + None, + ) + current = ( + _string_attr(parent_event, "parent_request_alias") + if parent_event is not None + else None + ) + return lineage + + +def _string_attr(event: TimelineEventArtifact, name: str) -> str | None: + value = event.attributes.get(name) + return value if isinstance(value, str) else None + + +def _required_attr(event: TimelineEventArtifact, name: str) -> str: + value = _string_attr(event, name) + if value is None: # ServiceEvent validation makes this unreachable. + raise ValueError(f"{event.event_id} is missing {name}") + return value + + +def _epoch(event: TimelineEventArtifact) -> int: + value = event.attributes.get("generation_epoch", 0) + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _csv_attr(event: TimelineEventArtifact, name: str) -> list[str]: + value = _string_attr(event, name) + return value.split(",") if value else [] diff --git a/src/voxbench/observability/observer.py b/src/voxbench/observability/observer.py index dc9a853..aaecbb0 100644 --- a/src/voxbench/observability/observer.py +++ b/src/voxbench/observability/observer.py @@ -14,6 +14,8 @@ from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen +from voxbench.observability.service_events import ServiceEvent + SipDirection = Literal["in", "out"] RtpDirection = Literal["received", "sent"] PCM16_FULL_SCALE = 32768.0 @@ -351,6 +353,7 @@ class ObservationBatch: sip_events: tuple[SipEvent, ...] = () rtp_stats: tuple[RtpStats, ...] = () timeline_events: tuple[TimelineEvent, ...] = () + service_events: tuple[ServiceEvent, ...] = () def to_payload(self) -> dict[str, Any]: return { @@ -360,6 +363,7 @@ def to_payload(self) -> dict[str, Any]: "sip_events": [event.to_payload() for event in self.sip_events], "rtp_stats": [stats.to_payload() for stats in self.rtp_stats], "timeline_events": [event.to_payload() for event in self.timeline_events], + "service_events": [event.to_payload() for event in self.service_events], } @property @@ -370,6 +374,7 @@ def item_count(self) -> int: + len(self.sip_events) + len(self.rtp_stats) + len(self.timeline_events) + + len(self.service_events) ) @@ -379,23 +384,37 @@ def item_count(self) -> int: "sip_events": 64, "rtp_stats": 64, "timeline_events": 128, + "service_events": 128, } def _bounded_observation_batches(batch: ObservationBatch) -> tuple[ObservationBatch, ...]: + event_items = tuple(("timeline", item) for item in batch.timeline_events) + tuple( + ("service", item) for item in batch.service_events + ) batch_count = max( 1, *( (len(getattr(batch, field_name)) + limit - 1) // limit for field_name, limit in OBSERVATION_BATCH_LIMITS.items() + if field_name not in {"timeline_events", "service_events"} ), + (len(event_items) + 127) // 128, ) batches: list[ObservationBatch] = [] for index in range(batch_count): - values = { + values: dict[str, Any] = { field_name: getattr(batch, field_name)[index * limit : (index + 1) * limit] for field_name, limit in OBSERVATION_BATCH_LIMITS.items() + if field_name not in {"timeline_events", "service_events"} } + event_chunk = event_items[index * 128 : (index + 1) * 128] + values["timeline_events"] = tuple( + item for item_type, item in event_chunk if item_type == "timeline" + ) + values["service_events"] = tuple( + item for item_type, item in event_chunk if item_type == "service" + ) candidate = ObservationBatch(run_id=batch.run_id, **values) if candidate.item_count: batches.append(candidate) @@ -413,6 +432,7 @@ def _merge_observation_batches( sip_events=tuple(item for batch in batches for item in batch.sip_events), rtp_stats=tuple(item for batch in batches for item in batch.rtp_stats), timeline_events=tuple(item for batch in batches for item in batch.timeline_events), + service_events=tuple(item for batch in batches for item in batch.service_events), ) @@ -470,14 +490,25 @@ def _request(self, method: str, path: str, payload: Mapping[str, Any]) -> dict[s class VoxBenchObserver: """Collect audio and telemetry without coupling to a provider or media framework.""" - def __init__(self, run_id: str, transport: ObservationTransport) -> None: + def __init__( + self, + run_id: str, + transport: ObservationTransport, + *, + max_pending_items: int = 2048, + ) -> None: + if not 1 <= max_pending_items <= 100_000: + raise ValueError("max_pending_items must be between 1 and 100000") self.run_id = run_id self.transport = transport + self.max_pending_items = max_pending_items + self._observation_drop_count = 0 self._metrics: list[MetricPoint] = [] self._audio_chunks: list[AudioChunk] = [] self._sip_events: list[SipEvent] = [] self._rtp_stats: list[RtpStats] = [] self._timeline_events: list[TimelineEvent] = [] + self._service_events: list[ServiceEvent] = [] self._rtp_packet_ordinal = 0 self._rtp_capture_health_ordinal = 0 self._discontinuity_ordinal = 0 @@ -559,7 +590,7 @@ def observe_stage_audio( ) ) with self._lock: - self._metrics.extend(metrics) + self._extend_pending(self._metrics, metrics) prior = self._stage_pcm_state.get(stage) same_format = ( prior is not None and prior[:2] == (sample_rate_hz, channels) @@ -581,7 +612,8 @@ def observe_stage_audio( ): event_ordinal = self._discontinuity_ordinal self._discontinuity_ordinal += 1 - self._timeline_events.append( + self._append_pending( + self._timeline_events, TimelineEvent( event_id=f"pcm-discontinuity:{event_ordinal}", category="pipeline", @@ -628,14 +660,15 @@ def observe_stage_audio( last_incident_ms, ) if record_output and output_pcm_s16le: - self._audio_chunks.append( + self._append_pending( + self._audio_chunks, AudioChunk( stage=stage, pcm_s16le=output_pcm_s16le, sample_rate_hz=sample_rate_hz, channels=channels, ts=observed_at, - ) + ), ) def observe_metric( @@ -647,17 +680,18 @@ def observe_metric( ts: datetime | None = None, ) -> None: with self._lock: - self._metrics.append( + self._append_pending( + self._metrics, MetricPoint(stage=stage, name=name, value=float(value), ts=ts or _utc_now()) ) def observe_sip_event(self, event: SipEvent) -> None: with self._lock: - self._sip_events.append(event) + self._append_pending(self._sip_events, event) def observe_rtp_stats(self, stats: RtpStats) -> None: with self._lock: - self._rtp_stats.append(stats) + self._append_pending(self._rtp_stats, stats) def observe_rtp_packet(self, packet: RtpPacket) -> None: """Record only safe RTP header/cadence evidence, never packet payload.""" @@ -665,7 +699,8 @@ def observe_rtp_packet(self, packet: RtpPacket) -> None: with self._lock: packet_ordinal = self._rtp_packet_ordinal self._rtp_packet_ordinal += 1 - self._timeline_events.append( + self._append_pending( + self._timeline_events, TimelineEvent( event_id=f"rtp-packet:{packet_ordinal}", category="transport", @@ -684,7 +719,7 @@ def observe_rtp_packet(self, packet: RtpPacket) -> None: "marker": packet.marker, }, ts=packet.ts, - ) + ), ) def observe_rtp_capture_health( @@ -696,7 +731,8 @@ def observe_rtp_capture_health( with self._lock: event_ordinal = self._rtp_capture_health_ordinal self._rtp_capture_health_ordinal += 1 - self._timeline_events.append( + self._append_pending( + self._timeline_events, TimelineEvent( event_id=f"rtp-capture-health:{event_ordinal}", category="transport", @@ -718,12 +754,21 @@ def observe_rtp_capture_health( "window_duration_ms": snapshot.window_duration_ms, }, ts=snapshot.ts, - ) + ), ) def observe_timeline_event(self, event: TimelineEvent) -> None: with self._lock: - self._timeline_events.append(event) + self._append_pending(self._timeline_events, event) + + def observe_service_event(self, event: ServiceEvent) -> None: + """Queue a safe typed service boundary without retaining conversation content.""" + + validated = ServiceEvent.model_validate( + event.model_dump(mode="python", exclude_none=True) + ) + with self._lock: + self._append_pending(self._service_events, validated) def flush(self) -> int: """Send pending observations and restore them if the transport fails.""" @@ -736,12 +781,14 @@ def flush(self) -> int: sip_events=tuple(self._sip_events), rtp_stats=tuple(self._rtp_stats), timeline_events=tuple(self._timeline_events), + service_events=tuple(self._service_events), ) self._metrics.clear() self._audio_chunks.clear() self._sip_events.clear() self._rtp_stats.clear() self._timeline_events.clear() + self._service_events.clear() if batch.item_count == 0: return 0 batches = _bounded_observation_batches(batch) @@ -752,11 +799,15 @@ def flush(self) -> int: except Exception: unsent = _merge_observation_batches(self.run_id, batches[index:]) with self._lock: + self._trim_pending_to_size( + max(0, self.max_pending_items - unsent.item_count) + ) self._metrics[0:0] = unsent.metrics self._audio_chunks[0:0] = unsent.audio_chunks self._sip_events[0:0] = unsent.sip_events self._rtp_stats[0:0] = unsent.rtp_stats self._timeline_events[0:0] = unsent.timeline_events + self._service_events[0:0] = unsent.service_events raise sent_count += bounded_batch.item_count return sent_count @@ -770,8 +821,57 @@ def pending_count(self) -> int: + len(self._sip_events) + len(self._rtp_stats) + len(self._timeline_events) + + len(self._service_events) ) + @property + def observation_drop_count(self) -> int: + with self._lock: + return self._observation_drop_count + + def _pending_count_unlocked(self) -> int: + return ( + len(self._metrics) + + len(self._audio_chunks) + + len(self._sip_events) + + len(self._rtp_stats) + + len(self._timeline_events) + + len(self._service_events) + ) + + def _append_pending(self, target: list[Any], item: Any) -> None: + if self._pending_count_unlocked() >= self.max_pending_items: + self._observation_drop_count += 1 + return + target.append(item) + + def _extend_pending(self, target: list[Any], items: list[Any]) -> None: + remaining = max(0, self.max_pending_items - self._pending_count_unlocked()) + target.extend(items[:remaining]) + self._observation_drop_count += len(items) - remaining + + def _trim_pending_to_size(self, maximum: int) -> None: + overflow = self._pending_count_unlocked() - maximum + if overflow <= 0: + return + # A send failure must restore already-admitted items. Drop newer arrivals + # from list tails first so the retry set stays bounded. + for target in ( + self._service_events, + self._timeline_events, + self._rtp_stats, + self._sip_events, + self._audio_chunks, + self._metrics, + ): + removed = min(overflow, len(target)) + if removed: + del target[-removed:] + self._observation_drop_count += removed + overflow -= removed + if overflow == 0: + break + class RtpPacketTapAdapter: """Project transient RTP datagrams and explicit capture health into an observer.""" diff --git a/src/voxbench/observability/service_events.py b/src/voxbench/observability/service_events.py new file mode 100644 index 0000000..a83c758 --- /dev/null +++ b/src/voxbench/observability/service_events.py @@ -0,0 +1,251 @@ +"""Closed, metadata-only lifecycle observations for voice AI services.""" + +from __future__ import annotations + +import math +import re +from datetime import UTC, datetime +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +ServiceEventKind = Literal[ + "service.session_started", + "service.session_ended", + "stt.partial_emitted", + "stt.final_emitted", + "turn.speech_ended", + "turn.committed", + "llm.request_started", + "llm.first_output", + "llm.first_answer_text", + "llm.response_completed", + "aggregation.segment_ready", + "tts.request_started", + "tts.first_pcm", + "tts.response_completed", + "realtime.request_started", + "realtime.first_audio", + "realtime.response_completed", + "playback.write_started", + "playback.write_completed", + "playback.discarded", + "cancel.requested", + "cancel.acknowledged", + "service.failed", +] +ServiceEventRole = Literal[ + "stt", "llm", "tts", "realtime", "aggregation", "coordinator", "playback" +] +RelationName = Literal[ + "session", "turn", "request", "parent_request", "response", "segment" +] +ServiceAttributeName = Literal[ + "attempt_ordinal", + "sequence_ordinal", + "duration_ms", + "audio_duration_ms", + "input_units", + "output_units", + "cached_input_units", + "tool_call_count", + "queue_depth", + "sample_rate_hz", + "channels", + "encoding", + "retry", + "truncated", + "terminal_outcome", + "failure_alias", + "cancel_scope", + "observed_boundary", + "output_kind", +] +Scalar = str | int | float | bool | None + +_ALIAS = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") +_EXPECTED_RELATIONS: dict[str, tuple[RelationName, ...]] = { + "service.session_started": ("session",), + "service.session_ended": ("session",), + "stt.partial_emitted": ("session",), + "stt.final_emitted": ("session", "request"), + "turn.speech_ended": ("turn", "request"), + "turn.committed": ("turn", "parent_request"), + "llm.request_started": ("turn", "request", "parent_request", "response"), + "llm.first_output": ("turn", "request", "response"), + "llm.first_answer_text": ("turn", "request", "response"), + "llm.response_completed": ("turn", "request", "response"), + "aggregation.segment_ready": ("request", "response", "segment", "parent_request"), + "tts.request_started": ("request", "parent_request", "response", "segment"), + "tts.first_pcm": ("request", "response", "segment"), + "tts.response_completed": ("request", "response", "segment"), + "realtime.request_started": ("turn", "request", "response"), + "realtime.first_audio": ("request", "response"), + "realtime.response_completed": ("request", "response"), + "playback.write_started": ("parent_request", "response", "segment"), + "playback.write_completed": ("parent_request", "response", "segment"), + "playback.discarded": ("parent_request", "response", "segment"), + "cancel.requested": ("request",), + "cancel.acknowledged": ("request",), + "service.failed": ("request",), +} +_KIND_ROLES: dict[str, frozenset[str]] = { + **{kind: frozenset({"stt"}) for kind in _EXPECTED_RELATIONS if kind.startswith("stt.")}, + **{kind: frozenset({"llm"}) for kind in _EXPECTED_RELATIONS if kind.startswith("llm.")}, + **{kind: frozenset({"tts"}) for kind in _EXPECTED_RELATIONS if kind.startswith("tts.")}, + **{ + kind: frozenset({"realtime"}) + for kind in _EXPECTED_RELATIONS + if kind.startswith("realtime.") + }, + "turn.committed": frozenset({"coordinator"}), + "turn.speech_ended": frozenset({"coordinator"}), + "aggregation.segment_ready": frozenset({"aggregation"}), + "playback.write_started": frozenset({"playback"}), + "playback.write_completed": frozenset({"playback"}), + "playback.discarded": frozenset({"playback"}), + "service.session_started": frozenset({"stt", "llm", "tts", "realtime"}), + "service.session_ended": frozenset({"stt", "llm", "tts", "realtime"}), + "cancel.requested": frozenset({"stt", "llm", "tts", "realtime", "playback"}), + "cancel.acknowledged": frozenset({"stt", "llm", "tts", "realtime", "playback"}), + "service.failed": frozenset( + {"stt", "llm", "tts", "realtime", "aggregation", "playback"} + ), +} + + +class ServiceEvent(BaseModel): + """One observed boundary with local aliases and no conversation content.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + schema_version: Literal["voxbench/service-event/v1"] = "voxbench/service-event/v1" + collector_alias: Annotated[str, Field(min_length=1, max_length=48)] + event_alias: Annotated[str, Field(min_length=1, max_length=48)] + kind: ServiceEventKind + component_id: Annotated[str, Field(min_length=1, max_length=64)] + role: ServiceEventRole + generation_epoch: int = Field(default=0, ge=0) + session_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + turn_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + request_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + parent_request_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + response_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + segment_alias: Annotated[str, Field(min_length=1, max_length=128)] | None = None + unobserved_relations: list[RelationName] = Field(default_factory=list, max_length=6) + attributes: dict[ServiceAttributeName, Scalar] = Field(default_factory=dict) + ts: datetime = Field(default_factory=lambda: datetime.now(UTC)) + clock_domain: Annotated[str, Field(min_length=1, max_length=64)] = "application_wall" + alignment_uncertainty_ms: float | None = Field(default=None, ge=0) + + @field_validator( + "collector_alias", "event_alias", "component_id", "session_alias", "turn_alias", + "request_alias", "parent_request_alias", "response_alias", "segment_alias", + "clock_domain", + ) + @classmethod + def validate_alias(cls, value: str | None, info) -> str | None: + if value is not None and not _ALIAS.fullmatch(value): + raise ValueError(f"{info.field_name} must be a local safe alias") + return value + + @field_validator("unobserved_relations") + @classmethod + def validate_missing_relations(cls, value: list[RelationName]) -> list[RelationName]: + if len(value) != len(set(value)): + raise ValueError("unobserved_relations must be unique") + return value + + @field_validator("attributes") + @classmethod + def validate_attributes( + cls, value: dict[ServiceAttributeName, Scalar], + ) -> dict[ServiceAttributeName, Scalar]: + result: dict[ServiceAttributeName, Scalar] = {} + integer_keys = { + "attempt_ordinal", "sequence_ordinal", "input_units", "output_units", + "cached_input_units", "tool_call_count", "queue_depth", "sample_rate_hz", + "channels", + } + number_keys = {"duration_ms", "audio_duration_ms"} + boolean_keys = {"retry", "truncated"} + for key, item in value.items(): + if item is None: + raise ValueError(f"attribute '{key}' must not be null") + if key in integer_keys and ( + isinstance(item, bool) or not isinstance(item, int) or item < 0 + ): + raise ValueError(f"attribute '{key}' must be a non-negative integer") + if key in number_keys and ( + isinstance(item, bool) + or not isinstance(item, int | float) + or not math.isfinite(item) + or item < 0 + ): + raise ValueError(f"attribute '{key}' must be finite and non-negative") + if key in boolean_keys and not isinstance(item, bool): + raise ValueError(f"attribute '{key}' must be boolean") + if key not in integer_keys | number_keys | boolean_keys and not isinstance(item, str): + raise ValueError(f"attribute '{key}' must be a safe alias string") + if isinstance(item, float) and not math.isfinite(item): + raise ValueError(f"attribute '{key}' must be finite") + if isinstance(item, str) and ( + not item or len(item) > 128 or not _ALIAS.fullmatch(item) + ): + raise ValueError(f"attribute '{key}' must be a local safe alias") + if key == "output_kind" and item not in { + "answer", "reasoning", "tool_call", "other" + }: + raise ValueError( + "attribute 'output_kind' must be answer, reasoning, tool_call, or other" + ) + result[key] = item + return result + + @field_validator("ts") + @classmethod + def require_aware_timestamp(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("ts must include a UTC offset") + return value.astimezone(UTC) + + @model_validator(mode="after") + def validate_relationships_and_budget(self) -> ServiceEvent: + if self.role not in _KIND_ROLES[self.kind]: + raise ValueError(f"{self.kind} is not compatible with role '{self.role}'") + relations = { + "session": self.session_alias, + "turn": self.turn_alias, + "request": self.request_alias, + "parent_request": self.parent_request_alias, + "response": self.response_alias, + "segment": self.segment_alias, + } + missing = set(self.unobserved_relations) + for relation in _EXPECTED_RELATIONS[self.kind]: + if relations[relation] is None and relation not in missing: + raise ValueError( + f"{self.kind} requires {relation}_alias or an explicit " + f"unobserved_relations entry" + ) + if relations[relation] is not None and relation in missing: + raise ValueError(f"{relation} cannot be both linked and unobserved") + normalized_scalar_count = ( + 2 + + sum(item is not None for item in relations.values()) + + bool(self.unobserved_relations) + + len(self.attributes) + ) + if normalized_scalar_count > 15: + raise ValueError("service event reserves one of at most 16 attributes for coverage") + return self + + @property + def normalized_event_id(self) -> str: + return f"service:{self.collector_alias}:{self.event_alias}" + + def to_payload(self) -> dict[str, Any]: + return self.model_dump(mode="json", exclude_none=True) + + +SERVICE_EVENT_KINDS = frozenset(_EXPECTED_RELATIONS) diff --git a/src/voxbench/registry/config_views.py b/src/voxbench/registry/config_views.py new file mode 100644 index 0000000..bd5bd89 --- /dev/null +++ b/src/voxbench/registry/config_views.py @@ -0,0 +1,67 @@ +"""Read versioned resolved configs without rewriting stored v1 objects.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from voxbench.registry.errors import ConfigValidationError + + +def ai_mode(config: dict[str, Any]) -> str: + return config["spec"]["ai"].get("mode", "realtime") + + +def service_components(config: dict[str, Any]) -> dict[str, dict[str, Any]]: + ai = config["spec"]["ai"] + if config.get("apiVersion", "voxbench/v1") == "voxbench/v1": + component = deepcopy(ai) + component["id"] = "ai" + component["plugin"] = component.pop("provider") + return {"realtime": component} + roles = ("realtime",) if ai["mode"] == "realtime" else ("stt", "llm", "tts") + return {role: deepcopy(ai[role]) for role in roles} + + +def safe_ai_components(config: dict[str, Any]) -> list[dict[str, str]]: + """Return stable component identity without params, prompts, tools, or endpoints.""" + + components = service_components(config) + if config.get("apiVersion", "voxbench/v1") == "voxbench/v2": + aggregation = config["spec"]["ai"].get("text_aggregation") + if isinstance(aggregation, dict): + components["aggregation"] = aggregation + result: list[dict[str, str]] = [] + for role, component in components.items(): + item = { + "role": role, + "component_id": component["id"], + "plugin": component["plugin"], + } + for name in ("manifest_version", "model"): + value = component.get(name) + if isinstance(value, str): + item[name] = value + result.append(item) + roles = ("realtime", "stt", "llm", "aggregation", "tts") + role_order = {role: index for index, role in enumerate(roles)} + return sorted(result, key=lambda item: role_order.get(item["role"], len(role_order))) + + +def pipeline_chains(config: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + media = config["spec"]["media"] + if config.get("apiVersion", "voxbench/v1") == "voxbench/v1": + return {"legacy": deepcopy(media["pipeline"])} + return { + direction: deepcopy(media[f"{direction}_pipeline"]) + for direction in ("input", "output") + } + + +def require_v1_workflow(config: dict[str, Any], *, operation: str) -> None: + version = config.get("apiVersion", "voxbench/v1") + if version != "voxbench/v1": + raise ConfigValidationError( + f"{operation} does not support {version} yet; v2 currently supports " + "configuration resolution only" + ) diff --git a/src/voxbench/registry/service.py b/src/voxbench/registry/service.py index eac3a93..290da8d 100644 --- a/src/voxbench/registry/service.py +++ b/src/voxbench/registry/service.py @@ -4,11 +4,13 @@ import json from collections.abc import Iterable +from copy import deepcopy from itertools import pairwise from pathlib import Path from typing import Any from jsonschema import Draft202012Validator +from jsonschema import SchemaError as JsonSchemaError from jsonschema import ValidationError as JsonSchemaValidationError from pydantic import ValidationError as PydanticValidationError @@ -20,7 +22,9 @@ from voxbench.registry.hashing import resolved_hash from voxbench.registry.merge import deep_merge from voxbench.registry.resolved import ResolvedConfig +from voxbench.registry.v2 import resolve_v2 from voxbench.schemas import CapabilityManifest, IoContract, PipelineStage, VoiceConfig +from voxbench.schemas_v2 import CapabilityManifestV2, PipelineStageV2, VoiceConfigV2 class RegistryService: @@ -28,6 +32,7 @@ class RegistryService: def __init__(self) -> None: self._manifests: dict[tuple[str, str], CapabilityManifest] = {} + self._v2_manifests: dict[tuple[str, str, str], CapabilityManifestV2] = {} self._configs: dict[str, dict[str, Any]] = {} @classmethod @@ -44,32 +49,56 @@ def from_files( service.register_config(load_json(config_path)) return service - def register_manifest(self, raw: dict[str, Any]) -> CapabilityManifest: + def register_manifest(self, raw: dict[str, Any]) -> CapabilityManifest | CapabilityManifestV2: try: - manifest = CapabilityManifest.model_validate(raw) + manifest = ( + CapabilityManifestV2.model_validate(raw) + if raw.get("apiVersion") == "voxbench/v2" + else CapabilityManifest.model_validate(raw) + ) Draft202012Validator.check_schema(manifest.param_schema) - except (PydanticValidationError, JsonSchemaValidationError) as exc: + except (PydanticValidationError, JsonSchemaValidationError, JsonSchemaError) as exc: raise ConfigValidationError(str(exc)) from exc - self._manifests[(manifest.kind, manifest.name)] = manifest + if isinstance(manifest, CapabilityManifestV2): + key = (manifest.kind, manifest.name, manifest.version) + previous = self._v2_manifests.get(key) + if previous is not None and previous != manifest: + raise ConfigValidationError( + "v2 manifest identity is immutable; publish a new version" + ) + self._v2_manifests[key] = manifest.model_copy(deep=True) + else: + self._manifests[(manifest.kind, manifest.name)] = manifest return manifest def register_config(self, raw: dict[str, Any]) -> None: - name = raw.get("meta", {}).get("name") + meta = raw.get("meta") + name = meta.get("name") if isinstance(meta, dict) else None if not isinstance(name, str) or not name: raise ConfigValidationError("config meta.name is required") - self._configs[name] = raw + self._configs[name] = deepcopy(raw) if raw.get("apiVersion") == "voxbench/v2" else raw def resolve_config(self, name: str) -> ResolvedConfig: raw = self._resolve_raw(name, seen=set()) try: - config = VoiceConfig.model_validate(raw) + config = ( + VoiceConfigV2.model_validate(raw) + if raw.get("apiVersion") == "voxbench/v2" + else VoiceConfig.model_validate(raw) + ) except PydanticValidationError as exc: raise ConfigValidationError(str(exc)) from exc - self._validate_cross_fields(config) - resolved = config.model_dump(mode="json", exclude_none=True) - self._materialize_manifest_defaults(resolved) + if isinstance(config, VoiceConfigV2): + resolved = resolve_v2( + config, lookup=self._v2_manifest, validate_params=self._validate_params, + validate_stage=self._validate_v2_stage, + ) + else: + self._validate_cross_fields(config) + resolved = config.model_dump(mode="json", exclude_none=True) + self._materialize_manifest_defaults(resolved) return ResolvedConfig( name=config.meta.name, resolved=resolved, @@ -91,6 +120,19 @@ def _resolve_raw(self, name: str, *, seen: set[str]) -> dict[str, Any]: raise ConfigValidationError(f"config '{name}' meta.parent must be a string") base = self._resolve_raw(parent, seen=seen | {name}) + if "voxbench/v2" in (base.get("apiVersion"), raw.get("apiVersion")): + if raw.get("apiVersion") != base.get("apiVersion"): + raise ConfigValidationError("cross-version overlay inheritance is forbidden") + base_spec, overlay_spec = base.get("spec"), raw.get("spec", {}) + if not isinstance(base_spec, dict) or not isinstance(overlay_spec, dict): + raise ConfigValidationError("v2 overlay spec must be an object") + base_ai, overlay_ai = base_spec.get("ai"), overlay_spec.get("ai", {}) + if not isinstance(base_ai, dict) or not isinstance(overlay_ai, dict): + raise ConfigValidationError("v2 overlay ai must be an object") + base_mode = base_ai.get("mode") + overlay_mode = overlay_ai.get("mode", base_mode) + if overlay_mode != base_mode: + raise ConfigValidationError("cross-mode overlay inheritance is forbidden") return deep_merge(base, raw) def _validate_cross_fields(self, config: VoiceConfig) -> None: @@ -157,8 +199,8 @@ def _validate_provider_caps( def _validate_host_capabilities( self, - stage: PipelineStage, - manifest: CapabilityManifest, + stage: PipelineStage | PipelineStageV2, + manifest: CapabilityManifest | CapabilityManifestV2, ) -> None: required = set(stage.requires_host_capability or manifest.requires_host_capability) available = set(stage.host_capabilities) @@ -171,8 +213,8 @@ def _validate_host_capabilities( def _validate_overrides( self, - stage: PipelineStage, - manifest: CapabilityManifest, + stage: PipelineStage | PipelineStageV2, + manifest: CapabilityManifest | CapabilityManifestV2, ) -> None: allowed = {override.target: override.type for override in manifest.allowed_overrides} for override in stage.overrides: @@ -188,7 +230,7 @@ def _validate_overrides( def _validate_params( self, label: str, - manifest: CapabilityManifest, + manifest: CapabilityManifest | CapabilityManifestV2, params: dict[str, Any], ) -> None: try: @@ -220,6 +262,20 @@ def _manifest(self, kind: str, name: str) -> CapabilityManifest: except KeyError as exc: raise ManifestNotFoundError(f"unknown {kind} manifest '{name}'") from exc + def _v2_manifest(self, kind: str, name: str, version: str) -> CapabilityManifestV2: + try: + return self._v2_manifests[(kind, name, version)] + except KeyError as exc: + raise ManifestNotFoundError( + f"unknown v2 {kind} manifest '{name}' version '{version}'" + ) from exc + + def _validate_v2_stage( + self, stage: PipelineStageV2, manifest: CapabilityManifestV2, + ) -> None: + self._validate_host_capabilities(stage, manifest) + self._validate_overrides(stage, manifest) + def load_json(path: str | Path) -> dict[str, Any]: with Path(path).open(encoding="utf-8") as handle: diff --git a/src/voxbench/registry/v2.py b/src/voxbench/registry/v2.py new file mode 100644 index 0000000..5e61e33 --- /dev/null +++ b/src/voxbench/registry/v2.py @@ -0,0 +1,161 @@ +"""Static v2 validation; declared capabilities never establish live readiness.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from voxbench.registry.errors import ConfigValidationError +from voxbench.registry.hashing import resolved_hash +from voxbench.schemas_v2 import ( + AudioFormat, + CapabilityManifestV2, + CascadeAiConfig, + ComponentConfig, + PipelineStageV2, + ServiceConfig, + VoiceConfigV2, +) + +ManifestLookup = Callable[[str, str, str], CapabilityManifestV2] +ParamValidator = Callable[[str, Any, dict[str, Any]], None] +StageValidator = Callable[[Any, Any], None] + + +def resolve_v2( + config: VoiceConfigV2, + *, + lookup: ManifestLookup, + validate_params: ParamValidator, + validate_stage: StageValidator, +) -> dict[str, Any]: + spec = config.spec + engine = lookup("engine", spec.engine.kind, spec.engine.manifest_version) + validate_params("engine", engine, spec.engine.params) + caps = engine.engine_caps + assert caps is not None # Manifest kind validation establishes this contract. + if spec.transport.codec not in caps.supported_codecs: + raise ConfigValidationError("transport.codec is not supported by engine decode contract") + if spec.transport.ptime_ms <= 0: + raise ConfigValidationError("transport.ptime_ms must be positive") + + resolved = config.model_dump(mode="json", exclude_none=True) + pins = [_pin("engine", engine)] + authorities = {"application": {"speech_detector", "end_of_turn", "interruption"}} + authorities["engine"] = set(caps.authorities) + ids = {"application", "engine"} + + def register(component: ComponentConfig, kind: str) -> CapabilityManifestV2: + if component.id in ids: + raise ConfigValidationError(f"duplicate or reserved component id '{component.id}'") + ids.add(component.id) + manifest = lookup(kind, component.plugin, component.manifest_version) + validate_params(f"component '{component.id}'", manifest, component.params) + pins.append(_pin(component.id, manifest)) + authorities[component.id] = set(manifest.authorities) + return manifest + + services: dict[str, ServiceConfig] + if isinstance(spec.ai, CascadeAiConfig): + services = {role: getattr(spec.ai, role) for role in ("stt", "llm", "tts")} + aggregation = register(spec.ai.text_aggregation, "processor") + if aggregation.text_io is None: + raise ConfigValidationError("text aggregation requires a text→text processor") + if aggregation.requires_host_capability: + raise ConfigValidationError("text aggregation host capabilities are not declared") + else: + services = {"realtime": spec.ai.realtime} + + for role, service in services.items(): + manifest = register(service, "provider") + service_caps = manifest.service_caps + assert service_caps is not None + if role not in service_caps.roles: + raise ConfigValidationError(f"component '{service.id}' does not support role '{role}'") + if service_caps.supported_models and service.model not in service_caps.supported_models: + raise ConfigValidationError(f"component '{service.id}' model is not supported") + authorities[service.id] = set(service_caps.authorities) + for field, formats, modality in ( + ("input_audio", service_caps.audio_inputs, service_caps.input_modality), + ("output_audio", service_caps.audio_outputs, service_caps.output_modality), + ): + selected = getattr(service, field) + if modality == "audio" and (selected is None or selected not in formats): + raise ConfigValidationError( + f"component '{service.id}' {field} must select a supported audio format" + ) + if modality == "text" and selected is not None: + raise ConfigValidationError( + f"component '{service.id}' has audio on a text boundary" + ) + + input_service = services.get("stt") or services["realtime"] + output_service = services.get("tts") or services["realtime"] + assert input_service.input_audio is not None and output_service.output_audio is not None + + for direction, stages, start, end in ( + ("input", spec.media.input_pipeline, caps.decoded_input, input_service.input_audio), + ("output", spec.media.output_pipeline, output_service.output_audio, caps.decoded_output), + ): + current: AudioFormat = start + for index, stage in enumerate(stages): + manifest = register(stage, "processor") + io = manifest.io + if io is None: + raise ConfigValidationError(f"PCM stage '{stage.id}' requires audio IO") + if stage.io is not None and stage.io != io: + raise ConfigValidationError(f"stage '{stage.id}' cannot replace manifest IO") + validate_stage(stage, manifest) + _validate_stage_contract(stage, manifest) + if current != io.accepts: + raise ConfigValidationError( + f"{direction} pipeline io mismatch before stage '{stage.id}'" + ) + current = io.produces + target = resolved["spec"]["media"][f"{direction}_pipeline"][index] + target["io"] = io.model_dump(mode="json") + for field in ( + "invariants_enforced", "invariants_applicable", "lossy_expected", + "requires_host_capability", + ): + if field not in target: + target[field] = list(getattr(manifest, field)) + if current != end: + raise ConfigValidationError( + f"{direction} pipeline boundary format mismatch; add an explicit converter" + ) + + for decision, component_id in spec.turn_taking.model_dump(exclude_none=True).items(): + if decision not in authorities.get(component_id, set()): + raise ConfigValidationError( + f"turn authority '{component_id}' does not support '{decision}'" + ) + + resolved["manifest_pins"] = sorted(pins, key=lambda pin: pin["component_id"]) + return resolved + + +def _pin(component_id: str, manifest: CapabilityManifestV2) -> dict[str, str]: + return { + "component_id": component_id, + "kind": manifest.kind, + "name": manifest.name, + "version": manifest.version, + "digest": resolved_hash(manifest.model_dump(mode="json", exclude_none=True)), + } + + +def _validate_stage_contract(stage: PipelineStageV2, manifest: CapabilityManifestV2) -> None: + # A caller cannot remove actual host requirements or claim enforcement that + # the pinned implementation does not advertise. + targets = [override.target for override in stage.overrides] + if len(targets) != len(set(targets)): + raise ConfigValidationError(f"stage '{stage.id}' has duplicate override targets") + if stage.requires_host_capability is not None and not set( + manifest.requires_host_capability + ).issubset(stage.requires_host_capability): + raise ConfigValidationError(f"stage '{stage.id}' cannot remove host requirements") + if stage.invariants_enforced is not None and not set(stage.invariants_enforced).issubset( + manifest.invariants_enforced + ): + raise ConfigValidationError(f"stage '{stage.id}' cannot invent enforced invariants") diff --git a/src/voxbench/schemas_export.py b/src/voxbench/schemas_export.py index 8f305d6..2b155c6 100644 --- a/src/voxbench/schemas_export.py +++ b/src/voxbench/schemas_export.py @@ -5,13 +5,21 @@ import json from pathlib import Path +from voxbench.observability.service_events import ServiceEvent from voxbench.schemas import CapabilityManifest, VoiceConfig +from voxbench.schemas_v2 import CapabilityManifestV2, VoiceConfigV2 def export_schemas(output_dir: Path) -> None: output_dir.mkdir(parents=True, exist_ok=True) _write_schema(output_dir / "config.schema.json", VoiceConfig.model_json_schema()) _write_schema(output_dir / "manifest.schema.json", CapabilityManifest.model_json_schema()) + _write_schema(output_dir / "config.v2.schema.json", VoiceConfigV2.model_json_schema()) + _write_schema(output_dir / "manifest.v2.schema.json", CapabilityManifestV2.model_json_schema()) + _write_schema( + output_dir / "service-event.v1.schema.json", + ServiceEvent.model_json_schema(), + ) def _write_schema(path: Path, schema: dict[str, object]) -> None: @@ -20,4 +28,3 @@ def _write_schema(path: Path, schema: dict[str, object]) -> None: if __name__ == "__main__": export_schemas(Path("schemas")) - diff --git a/src/voxbench/schemas_v2.py b/src/voxbench/schemas_v2.py new file mode 100644 index 0000000..70ff068 --- /dev/null +++ b/src/voxbench/schemas_v2.py @@ -0,0 +1,264 @@ +"""Framework-independent v2 voice configuration and capability contracts. + +Keep these separate from v1: adding defaults to a legacy model changes resolved +JSON and the identity of previously published experiments. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field, model_validator + +from voxbench.schemas import ( + ConfigMeta, + Invariant, + JsonObject, + ObservabilityConfig, + OverridePermission, + OverrideValue, + PluginKind, + StrictModel, + TransportConfig, +) + +ComponentId = Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,63}$")] +NonEmpty = Annotated[str, Field(min_length=1)] +ServiceRole = Literal["stt", "llm", "tts", "realtime"] +Modality = Literal["audio", "text"] +Authority = Literal["speech_detector", "end_of_turn", "interruption"] +ServiceLatencyMeasurement = Literal[ + "stt_finalization_wait", + "turn_coordination_wait", + "llm_dispatch_wait", + "llm_first_output_wait", + "llm_first_answer_text_wait", + "text_aggregation_wait", + "tts_dispatch_queue_wait", + "tts_first_audio_wait", + "output_playback_start_wait", + "end_to_end_local_response_wait", +] + + +class AudioFormat(StrictModel): + encoding: Literal["pcm16", "pcm24", "pcm32", "float32"] + rate: int = Field(gt=0, strict=True) + channels: int = Field(gt=0, strict=True) + + +class AudioIoContract(StrictModel): + mode: Literal["passthrough", "rate_changing", "format_changing"] + accepts: AudioFormat + produces: AudioFormat + + @model_validator(mode="after") + def validate_mode(self) -> AudioIoContract: + if self.mode == "passthrough" and self.accepts != self.produces: + raise ValueError("passthrough IO must preserve the audio format") + if self.mode == "rate_changing" and ( + self.accepts.encoding != self.produces.encoding + or self.accepts.channels != self.produces.channels + ): + raise ValueError("rate_changing IO may change only the sample rate") + return self + + +class EngineCaps(StrictModel): + supported_codecs: list[NonEmpty] = Field(min_length=1) + decoded_input: AudioFormat + decoded_output: AudioFormat + authorities: list[Authority] = Field(default_factory=list) + + +class LifecycleCaps(StrictModel): + partial_output: bool = False + final_output: bool = False + response_boundaries: bool = False + usage: bool = False + + +class CancellationCaps(StrictModel): + request: bool = False + terminal_ack: bool = False + transport_abort: bool = False + conversation_truncation: bool = False + + +class ServiceCaps(StrictModel): + roles: list[ServiceRole] = Field(min_length=1) + input_modality: Modality + output_modality: Modality + audio_inputs: list[AudioFormat] = Field(default_factory=list) + audio_outputs: list[AudioFormat] = Field(default_factory=list) + lifecycle: LifecycleCaps = Field(default_factory=LifecycleCaps) + cancellation: CancellationCaps = Field(default_factory=CancellationCaps) + authorities: list[Authority] = Field(default_factory=list) + supported_models: list[NonEmpty] = Field(default_factory=list) + model_selection: Literal["explicit", "deployment_pinned"] = "explicit" + + @model_validator(mode="after") + def validate_modalities(self) -> ServiceCaps: + modalities = { + "stt": ("audio", "text"), + "llm": ("text", "text"), + "tts": ("text", "audio"), + "realtime": ("audio", "audio"), + } + if len(set(self.roles)) != len(self.roles): + raise ValueError("service roles must be unique") + for role in self.roles: + if (self.input_modality, self.output_modality) != modalities[role]: + raise ValueError(f"service modality mismatch for role '{role}'") + for modality, formats in ( + (self.input_modality, self.audio_inputs), + (self.output_modality, self.audio_outputs), + ): + if (modality == "audio") != bool(formats): + raise ValueError("audio boundaries require formats; text boundaries forbid them") + if "tts" in self.roles and self.authorities: + raise ValueError("TTS cannot own caller speech, end-of-turn or interruption decisions") + if "llm" in self.roles and "speech_detector" in self.authorities: + raise ValueError("text LLM cannot own caller speech detection") + return self + + +class TextIoContract(StrictModel): + input_modality: Literal["text"] + output_modality: Literal["text"] + + +class CapabilityManifestV2(StrictModel): + apiVersion: Literal["voxbench/v2"] + kind: PluginKind + name: NonEmpty + version: NonEmpty + param_schema: JsonObject = Field(default_factory=dict) + io: AudioIoContract | None = None + text_io: TextIoContract | None = None + engine_caps: EngineCaps | None = None + service_caps: ServiceCaps | None = None + authorities: list[Authority] = Field(default_factory=list) + invariants_enforced: list[Invariant] = Field(default_factory=list) + invariants_applicable: list[Invariant] = Field(default_factory=list) + lossy_expected: list[str] = Field(default_factory=list) + requires_host_capability: list[str] = Field(default_factory=list) + allowed_overrides: list[OverridePermission] = Field(default_factory=list) + + @model_validator(mode="after") + def validate_kind(self) -> CapabilityManifestV2: + targets = [override.target for override in self.allowed_overrides] + if len(targets) != len(set(targets)): + raise ValueError("manifest allowed override targets must be unique") + if self.kind == "engine": + if self.engine_caps is None or self.service_caps or self.io or self.text_io: + raise ValueError("engine manifest must declare only engine_caps") + elif self.kind == "provider": + if self.service_caps is None or self.engine_caps or self.io or self.text_io: + raise ValueError("provider manifest must declare only service_caps") + elif self.engine_caps or self.service_caps or (self.io is None) == (self.text_io is None): + raise ValueError("processor manifest must declare exactly one audio IO or text IO") + if self.text_io and ( + self.invariants_enforced or self.invariants_applicable or self.lossy_expected + ): + raise ValueError("text processors cannot declare audio invariants") + if self.kind != "processor" and self.authorities: + raise ValueError("engine/provider authorities belong inside their capability contract") + return self + + +class EngineConfigV2(StrictModel): + kind: NonEmpty + manifest_version: NonEmpty + params: JsonObject = Field(default_factory=dict) + + +class ComponentConfig(StrictModel): + id: ComponentId + plugin: NonEmpty + manifest_version: NonEmpty + params: JsonObject = Field(default_factory=dict) + + +class ServiceConfig(ComponentConfig): + model: NonEmpty + input_audio: AudioFormat | None = None + output_audio: AudioFormat | None = None + + +class ConversationServiceConfig(ServiceConfig): + system_prompt_ref: NonEmpty | None = None + tools: list[JsonObject] = Field(default_factory=list) + + +class RealtimeAiConfig(StrictModel): + mode: Literal["realtime"] + realtime: ConversationServiceConfig + + +class CascadeAiConfig(StrictModel): + mode: Literal["cascade"] + stt: ServiceConfig + llm: ConversationServiceConfig + text_aggregation: ComponentConfig + tts: ServiceConfig + + +class PipelineStageV2(ComponentConfig): + type: NonEmpty + io: AudioIoContract | None = None + invariants_enforced: list[Invariant] | None = None + invariants_applicable: list[Invariant] | None = None + lossy_expected: list[str] | None = None + requires_host_capability: list[str] | None = None + host_capabilities: list[str] = Field(default_factory=list) + overrides: list[OverrideValue] = Field(default_factory=list) + + +class MediaConfigV2(StrictModel): + input_pipeline: list[PipelineStageV2] + output_pipeline: list[PipelineStageV2] + + +class TurnTakingConfigV2(StrictModel): + # "application" is the application-owned coordinator; "engine" refers to + # the declared engine. Other references must identify configured components. + speech_detector: ComponentId + end_of_turn: ComponentId + interruption: ComponentId | None = None + + +class ServiceLatencySloV2(StrictModel): + id: ComponentId + measurement: ServiceLatencyMeasurement + max_ms: float = Field(gt=0, allow_inf_nan=False) + + +class ObservabilityConfigV2(ObservabilityConfig): + service_latency_slos: list[ServiceLatencySloV2] = Field(default_factory=list) + + @model_validator(mode="after") + def validate_service_latency_slos(self) -> ObservabilityConfigV2: + ids = [item.id for item in self.service_latency_slos] + measurements = [item.measurement for item in self.service_latency_slos] + if len(ids) != len(set(ids)): + raise ValueError("service latency SLO IDs must be unique") + if len(measurements) != len(set(measurements)): + raise ValueError("service latency SLO measurements must be unique") + return self + + +class VoiceSpecV2(StrictModel): + engine: EngineConfigV2 + transport: TransportConfig + media: MediaConfigV2 + turn_taking: TurnTakingConfigV2 + ai: Annotated[RealtimeAiConfig | CascadeAiConfig, Field(discriminator="mode")] + observability: ObservabilityConfigV2 = Field(default_factory=ObservabilityConfigV2) + + +class VoiceConfigV2(StrictModel): + apiVersion: Literal["voxbench/v2"] + kind: Literal["VoiceConfig"] + meta: ConfigMeta + spec: VoiceSpecV2 diff --git a/src/voxbench/synthetic_caller/offline.py b/src/voxbench/synthetic_caller/offline.py index bfd9ca4..f43a38d 100644 --- a/src/voxbench/synthetic_caller/offline.py +++ b/src/voxbench/synthetic_caller/offline.py @@ -13,6 +13,7 @@ from voxbench.engine_harness.models import MetricArtifact, RecordingArtifact from voxbench.engine_harness.plan import build_stage_plan from voxbench.media import mulaw_to_pcm16le, pcm16le_to_mulaw +from voxbench.registry.config_views import require_v1_workflow @dataclass(frozen=True) @@ -62,6 +63,7 @@ def generate_synthetic_artifacts( ) -> SyntheticArtifacts: """Generate local WAV recordings and cadence metrics for a resolved config.""" + require_v1_workflow(resolved_config, operation="legacy synthetic audio generation") _validate_audio_spec(audio_spec) degradations = degradations or {} output_root.mkdir(parents=True, exist_ok=True) diff --git a/src/voxbench/verification/core.py b/src/voxbench/verification/core.py index 49a271a..6c3b009 100644 --- a/src/voxbench/verification/core.py +++ b/src/voxbench/verification/core.py @@ -11,6 +11,7 @@ from urllib.parse import unquote, urlparse from voxbench.engine_harness.models import MetricArtifact, RecordingArtifact +from voxbench.registry.config_views import require_v1_workflow SUPPORTED_INVARIANTS = {"duration_preserving", "level_preserving", "isochronous"} DEFAULT_DURATION_TOLERANCE_RATIO = 0.02 @@ -49,6 +50,7 @@ def verify_recordings( ) -> list[VerificationResult]: """Verify declared signal invariants across adjacent stage recordings.""" + require_v1_workflow(resolved_config, operation="legacy recording verification") by_stage = {recording.stage: recording for recording in recordings} metrics_by_stage = _metrics_by_stage(metrics or []) previous: RecordingArtifact | None = None diff --git a/tests/fixtures/configs/v1-baseline.resolved.json b/tests/fixtures/configs/v1-baseline.resolved.json new file mode 100644 index 0000000..96f676d --- /dev/null +++ b/tests/fixtures/configs/v1-baseline.resolved.json @@ -0,0 +1 @@ +{"apiVersion":"voxbench/v1","kind":"VoiceConfig","meta":{"labels":{"env":"dev"},"name":"baseline","version":"1.0.0"},"spec":{"ai":{"model":"gemini-live-example","params":{"api_key_ref":"secret://gemini","response_modalities":["audio"]},"provider":"gemini","system_prompt_ref":"default@1.0.0","tools":[]},"engine":{"kind":"asterisk","params":{"media_profile":"8khz-mulaw-20ms","websocket_url":"ws://asterisk.example.invalid/media"}},"media":{"pipeline":[{"host_capabilities":["cadence_pacing"],"invariants_applicable":["duration_preserving","isochronous"],"invariants_enforced":["duration_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":24000},"mode":"rate_changing","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"input_rate":24000,"output_rate":8000,"quality":"high"},"plugin":"soxr-stream","requires_host_capability":["cadence_pacing"],"type":"resampler"},{"host_capabilities":[],"invariants_applicable":["level_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"passthrough","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"max_gain":8.0,"noise_floor":200,"target_rms":3000},"plugin":"rms-target","type":"agc"},{"host_capabilities":[],"invariants_applicable":["level_preserving"],"invariants_enforced":["level_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"passthrough","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"ceiling":0.7},"plugin":"peak-limiter","type":"limiter"},{"host_capabilities":["cadence_pacing","output_buffering"],"invariants_applicable":["duration_preserving","isochronous"],"invariants_enforced":["isochronous"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"format_changing","produces":{"channels":1,"encoding":"mulaw","rate":8000}},"lossy_expected":["bandwidth_limit"],"overrides":[],"params":{"encoding":"mulaw","ptime_ms":20},"plugin":"asterisk-ws","requires_host_capability":["cadence_pacing","output_buffering"],"type":"serializer"}]},"observability":{"cross_session":["task_count_trend","mem_trend"],"host_metrics":["cpu","active_tasks","loop_lag"],"record_audio":true,"signal_metrics":["rms_io","duration_ratio","frame_cadence_jitter"],"stage_taps":true,"trace_sample":1.0},"transport":{"codec":"pcm16","jitter_buffer":{"max_ms":200,"mode":"adaptive"},"ptime_ms":20},"turn_taking":{"barge_in":true,"owner":"server_vad"}}} diff --git a/tests/fixtures/configs/v1-overlay.resolved.json b/tests/fixtures/configs/v1-overlay.resolved.json new file mode 100644 index 0000000..64c4183 --- /dev/null +++ b/tests/fixtures/configs/v1-overlay.resolved.json @@ -0,0 +1 @@ +{"apiVersion":"voxbench/v1","kind":"VoiceConfig","meta":{"labels":{"env":"dev"},"name":"overlay-golden","parent":"baseline","version":"1.0.1"},"spec":{"ai":{"model":"pinned-overlay-model","params":{"api_key_ref":"secret://gemini","response_modalities":["audio"]},"provider":"gemini","system_prompt_ref":"default@1.0.0","tools":[]},"engine":{"kind":"asterisk","params":{"media_profile":"8khz-mulaw-20ms","websocket_url":"ws://asterisk.example.invalid/media"}},"media":{"pipeline":[{"host_capabilities":["cadence_pacing"],"invariants_applicable":["duration_preserving","isochronous"],"invariants_enforced":["duration_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":24000},"mode":"rate_changing","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"input_rate":24000,"output_rate":8000,"quality":"high"},"plugin":"soxr-stream","requires_host_capability":["cadence_pacing"],"type":"resampler"},{"host_capabilities":[],"invariants_applicable":["level_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"passthrough","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"max_gain":8.0,"noise_floor":200,"target_rms":3000},"plugin":"rms-target","type":"agc"},{"host_capabilities":[],"invariants_applicable":["level_preserving"],"invariants_enforced":["level_preserving"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"passthrough","produces":{"channels":1,"encoding":"pcm16","rate":8000}},"overrides":[],"params":{"ceiling":0.7},"plugin":"peak-limiter","type":"limiter"},{"host_capabilities":["cadence_pacing","output_buffering"],"invariants_applicable":["duration_preserving","isochronous"],"invariants_enforced":["isochronous"],"io":{"accepts":{"channels":1,"encoding":"pcm16","rate":8000},"mode":"format_changing","produces":{"channels":1,"encoding":"mulaw","rate":8000}},"lossy_expected":["bandwidth_limit"],"overrides":[],"params":{"encoding":"mulaw","ptime_ms":20},"plugin":"asterisk-ws","requires_host_capability":["cadence_pacing","output_buffering"],"type":"serializer"}]},"observability":{"cross_session":["task_count_trend","mem_trend"],"host_metrics":["cpu","active_tasks","loop_lag"],"record_audio":true,"signal_metrics":["rms_io","duration_ratio","frame_cadence_jitter"],"stage_taps":true,"trace_sample":1.0},"transport":{"codec":"pcm16","jitter_buffer":{"max_ms":200,"mode":"adaptive"},"ptime_ms":20},"turn_taking":{"barge_in":true,"owner":"server_vad"}}} diff --git a/tests/test_cascade_analysis.py b/tests/test_cascade_analysis.py new file mode 100644 index 0000000..0973cbe --- /dev/null +++ b/tests/test_cascade_analysis.py @@ -0,0 +1,548 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from voxbench.control_plane.app import create_app +from voxbench.control_plane.models import Base +from voxbench.control_plane.run_api import PostgresRunRepository +from voxbench.observability import ServiceEvent +from voxbench.registry.service import load_json + +ROOT = Path(__file__).resolve().parents[1] +V2 = ROOT / "examples/v2" + + +def _run_payload(*, latency_slos: list[dict] | None = None) -> dict: + config = load_json(V2 / "configs/cascade.json") + if latency_slos is not None: + config["spec"]["observability"] = {"service_latency_slos": latency_slos} + return { + "config_name": config["meta"]["name"], + "configs": [config], + "manifests": [ + load_json(filename) + for filename in sorted((V2 / "manifests").glob("*.json")) + ], + "call_id": "cascade-analysis", + } + + +def _start( + client: TestClient, + *, + latency_slos: list[dict] | None = None, +) -> tuple[str, dict]: + response = client.post( + "/runs/observed", + json=_run_payload(latency_slos=latency_slos), + ) + assert response.status_code == 200, response.text + return response.json()["run_id"], response.json() + + +def _event( + kind: str, + component_id: str, + role: str, + alias: str, + at_ms: int, + *, + epoch: int = 0, + clock_domain: str = "application-monotonic-a", + uncertainty_ms: float | None = 1.0, + attributes: dict | None = None, + **relations, +) -> ServiceEvent: + return ServiceEvent( + collector_alias="analysis-fixture", + event_alias=alias, + kind=kind, + component_id=component_id, + role=role, + generation_epoch=epoch, + ts=datetime(2026, 9, 27, 2, tzinfo=UTC) + timedelta(milliseconds=at_ms), + clock_domain=clock_domain, + alignment_uncertainty_ms=uncertainty_ms, + attributes=attributes or {}, + **relations, + ) + + +def _complete_path( + prefix: str = "one", + *, + epoch: int = 0, + include_terminals: bool = True, +) -> list[ServiceEvent]: + turn = f"turn-{prefix}" + stt = f"stt-{prefix}" + llm = f"llm-{prefix}" + response = f"response-{prefix}" + events = [ + _event( + "turn.speech_ended", + "application", + "coordinator", + f"{prefix}-speech-end", + 100, + epoch=epoch, + turn_alias=turn, + request_alias=stt, + ), + _event( + "stt.final_emitted", + "stt-main", + "stt", + f"{prefix}-stt-final", + 140, + epoch=epoch, + session_alias="stt-session", + request_alias=stt, + ), + _event( + "turn.committed", + "application", + "coordinator", + f"{prefix}-turn", + 150, + epoch=epoch, + turn_alias=turn, + parent_request_alias=stt, + ), + _event( + "llm.request_started", + "llm-main", + "llm", + f"{prefix}-llm-start", + 160, + epoch=epoch, + turn_alias=turn, + request_alias=llm, + parent_request_alias=stt, + response_alias=response, + ), + _event( + "llm.first_output", + "llm-main", + "llm", + f"{prefix}-llm-output", + 180, + epoch=epoch, + turn_alias=turn, + request_alias=llm, + response_alias=response, + attributes={"output_kind": "reasoning"}, + ), + _event( + "llm.first_answer_text", + "llm-main", + "llm", + f"{prefix}-llm-answer", + 200, + epoch=epoch, + turn_alias=turn, + request_alias=llm, + response_alias=response, + ), + ] + segment_timings = ((0, 220, 230, 260, 280), (1, 240, 245, 275, 300)) + for ordinal, ready_ms, start_ms, pcm_ms, write_ms in segment_timings: + segment = f"segment-{prefix}-{ordinal}" + aggregation = f"aggregation-{prefix}-{ordinal}" + tts = f"tts-{prefix}-{ordinal}" + events.extend( + [ + _event( + "aggregation.segment_ready", + "aggregation-main", + "aggregation", + f"{prefix}-ready-{ordinal}", + ready_ms, + epoch=epoch, + request_alias=aggregation, + parent_request_alias=llm, + response_alias=response, + segment_alias=segment, + attributes={"sequence_ordinal": ordinal}, + ), + _event( + "tts.request_started", + "tts-main", + "tts", + f"{prefix}-tts-start-{ordinal}", + start_ms, + epoch=epoch, + request_alias=tts, + parent_request_alias=aggregation, + response_alias=response, + segment_alias=segment, + ), + _event( + "tts.first_pcm", + "tts-main", + "tts", + f"{prefix}-tts-pcm-{ordinal}", + pcm_ms, + epoch=epoch, + request_alias=tts, + response_alias=response, + segment_alias=segment, + ), + _event( + "playback.write_started", + "output-resampler", + "playback", + f"{prefix}-write-{ordinal}", + write_ms, + epoch=epoch, + parent_request_alias=tts, + response_alias=response, + segment_alias=segment, + ), + ] + ) + if include_terminals: + events.extend( + [ + _event( + "tts.response_completed", + "tts-main", + "tts", + f"{prefix}-tts-done-{ordinal}", + pcm_ms + 80, + epoch=epoch, + request_alias=tts, + response_alias=response, + segment_alias=segment, + attributes={"terminal_outcome": "completed"}, + ), + _event( + "playback.write_completed", + "output-resampler", + "playback", + f"{prefix}-write-done-{ordinal}", + write_ms + 100, + epoch=epoch, + parent_request_alias=tts, + response_alias=response, + segment_alias=segment, + attributes={"audio_duration_ms": 100}, + ), + ] + ) + if include_terminals: + events.append( + _event( + "llm.response_completed", + "llm-main", + "llm", + f"{prefix}-llm-done", + 360, + epoch=epoch, + turn_alias=turn, + request_alias=llm, + response_alias=response, + attributes={"terminal_outcome": "completed"}, + ) + ) + return events + + +def _ingest(client: TestClient, run_id: str, events: list[ServiceEvent]) -> None: + response = client.post( + "/v1/observations", + json={ + "run_id": run_id, + "service_events": [event.model_dump(mode="json") for event in events], + }, + ) + assert response.status_code == 200, response.text + + +def _measurements(turn: dict) -> dict[str, list[dict]]: + result: dict[str, list[dict]] = {} + for measurement in turn["measurements"]: + result.setdefault(measurement["name"], []).append(measurement) + return result + + +def test_overlapping_multisegment_path_uses_correlated_boundaries_without_double_count( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + _ingest(client, run_id, _complete_path()) + + timeline = client.get(f"/runs/{run_id}/timeline").json() + assert len(timeline["lanes"]["turns"]) == 1 + turn = timeline["lanes"]["turns"][0] + measurements = _measurements(turn) + assert measurements["llm_first_output_wait"][0]["duration_ms"] == 20 + assert measurements["llm_first_answer_text_wait"][0]["duration_ms"] == 40 + assert [item["duration_ms"] for item in measurements["tts_first_audio_wait"]] == [30, 30] + end_to_end = measurements["end_to_end_local_response_wait"][0] + assert end_to_end["duration_ms"] == 180 + assert end_to_end["end_event_ref"] == "service:analysis-fixture:one-write-0" + assert turn["critical_path"] == { + "status": "complete", + "measurement_ids": turn["critical_path"]["measurement_ids"], + "end_to_end_measurement_id": end_to_end["measurement_id"], + "covered_duration_ms": 180.0, + "unexplained_duration_ms": 0.0, + "reason_alias": None, + } + assert turn["completion_observed"] is True + assert all( + item["definition_version"] == "cascade-latency/v1" + for item in turn["measurements"] + ) + assert all( + item["evidence_refs"] + for item in turn["measurements"] + if item["status"] == "observed" + ) + + derived = [ + interval + for interval in timeline["lanes"]["intervals"] + if interval["source"] == "cascade_causal_analysis_v1" + ] + assert any(item["name"] == "cascade.end_to_end_local_response_wait" for item in derived) + assert not any("response_completed" in item["name"] for item in derived) + assert not any( + item["rule_id"].startswith("cascade_latency") + for item in timeline["lanes"]["incidents"] + ) + + +def test_missing_speech_end_and_unknown_parent_are_explicitly_unobserved( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + events = [event for event in _complete_path() if event.kind != "turn.speech_ended"] + llm_index = next( + index + for index, event in enumerate(events) + if event.kind == "llm.request_started" + ) + events[llm_index] = events[llm_index].model_copy( + update={"parent_request_alias": "stt-parent-never-observed"} + ) + _ingest(client, run_id, events) + + turn = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"][0] + measurements = _measurements(turn) + assert measurements["stt_finalization_wait"][0]["status"] == "unobserved" + assert measurements["end_to_end_local_response_wait"][0]["status"] == "unobserved" + assert turn["critical_path"]["status"] == "incomplete" + assert any("parent_request" in item for item in turn["coverage"]["unresolved_relations"]) + assert turn["coverage"]["remote_playout_observed"] is False + + +def test_uncalibrated_cross_domain_latency_is_indeterminate(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + events = _complete_path() + index = next( + index + for index, event in enumerate(events) + if event.kind == "llm.first_answer_text" + ) + events[index] = events[index].model_copy(update={"clock_domain": "provider-wall-b"}) + _ingest(client, run_id, events) + + turn = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"][0] + answer = _measurements(turn)["llm_first_answer_text_wait"][0] + assert answer["status"] == "indeterminate" + assert answer["duration_ms"] is None + assert answer["reason_alias"] == "clock_domain_mismatch" + assert answer["evidence_refs"] == [ + "service:analysis-fixture:one-llm-start", + "service:analysis-fixture:one-llm-answer", + ] + assert turn["critical_path"]["status"] == "indeterminate" + + +def test_negative_endpoint_order_is_indeterminate_not_zero(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + events = _complete_path() + index = next(index for index, event in enumerate(events) if event.kind == "tts.first_pcm") + events[index] = events[index].model_copy( + update={"ts": datetime(2026, 9, 27, 2, tzinfo=UTC) + timedelta(milliseconds=225)} + ) + _ingest(client, run_id, events) + + turn = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"][0] + first_audio = _measurements(turn)["tts_first_audio_wait"][0] + assert first_audio["status"] == "indeterminate" + assert first_audio["reason_alias"] == "end_precedes_start" + assert first_audio["duration_ms"] is None + + +def test_new_epoch_does_not_reuse_old_response_playback(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + old_events = _complete_path("old", epoch=0) + new_events = [ + event + for event in _complete_path("new", epoch=1, include_terminals=False) + if not event.kind.startswith("aggregation.") + and not event.kind.startswith("tts.") + and not event.kind.startswith("playback.") + ] + _ingest(client, run_id, old_events + new_events) + + turns = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"] + new_turn = next(item for item in turns if item["generation_epoch"] == 1) + end_to_end = _measurements(new_turn)["end_to_end_local_response_wait"][0] + assert end_to_end["status"] == "unobserved" + assert end_to_end["end_event_ref"] is None + assert new_turn["critical_path"]["status"] == "incomplete" + + +def test_tool_continuation_is_a_separate_operation_with_explicit_parent_chain( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start(client) + first_operation = _complete_path("main") + continuation: list[ServiceEvent] = [] + for event in _complete_path("continuation"): + if event.kind in {"turn.speech_ended", "stt.final_emitted", "turn.committed"}: + continue + update: dict = {"ts": event.ts + timedelta(milliseconds=500)} + if event.turn_alias == "turn-continuation": + update["turn_alias"] = "turn-main" + if event.kind == "llm.request_started": + update["parent_request_alias"] = "llm-main" + continuation.append(event.model_copy(update=update)) + _ingest(client, run_id, first_operation + continuation) + + turns = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"] + assert [item["operation_request_alias"] for item in turns] == [ + "llm-continuation", + "llm-main", + ] + continued = next( + item for item in turns if item["operation_request_alias"] == "llm-continuation" + ) + end_to_end = _measurements(continued)["end_to_end_local_response_wait"][0] + assert end_to_end["status"] == "observed" + assert end_to_end["end_event_ref"] == ( + "service:analysis-fixture:continuation-write-0" + ) + assert end_to_end["duration_ms"] == 680 + + +def test_safe_ai_projection_is_consistent_and_excludes_component_params(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, created = _start(client) + expected_roles = ["stt", "llm", "aggregation", "tts"] + assert created["ai_mode"] == "cascade" + assert [item["role"] for item in created["ai_components"]] == expected_roles + encoded = json.dumps(created["ai_components"]) + assert "params" not in encoded + assert "system_prompt_ref" not in encoded + assert "tools" not in encoded + + recent = client.get("/runs").json()[0] + live = client.get("/runs/live-preview").json()[0] + timeline = client.get(f"/runs/{run_id}/timeline").json() + for projection in (recent, live, timeline): + assert projection["ai_mode"] == "cascade" + assert projection["ai_components"] == created["ai_components"] + + +def test_latency_incident_requires_and_cites_explicit_slo(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id, _ = _start( + client, + latency_slos=[{ + "id": "local-response", + "measurement": "end_to_end_local_response_wait", + "max_ms": 150, + }], + ) + _ingest(client, run_id, _complete_path()) + timeline = client.get(f"/runs/{run_id}/timeline").json() + incident = next( + item + for item in timeline["lanes"]["incidents"] + if item["rule_id"] == "cascade_latency_slo_v1" + ) + assert incident["observed"]["duration_ms"] == 180 + assert incident["expected"] == { + "service_latency_slo_id": "local-response", + "duration_ms_at_or_below": 150, + } + assert incident["evidence_refs"] == [ + "service:analysis-fixture:one-speech-end", + "service:analysis-fixture:one-write-0", + ] + interval = next( + item + for item in timeline["lanes"]["intervals"] + if item["name"] == "cascade.end_to_end_local_response_wait" + ) + assert interval["attributes"]["latency_threshold_status"] == "exceeded" + assert interval["attributes"]["latency_slo_id"] == "local-response" + + +def test_sql_repository_reload_reconstructs_identical_causal_analysis(tmp_path: Path) -> None: + engine = create_engine( + "sqlite+pysqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + repository = PostgresRunRepository(sessions) + first = TestClient(create_app(artifact_root=tmp_path, repository=repository)) + run_id, _ = _start(first) + _ingest(first, run_id, _complete_path()) + before = first.get(f"/runs/{run_id}/timeline").json()["lanes"]["turns"] + + restored = PostgresRunRepository(sessions).get(run_id) + assert restored is not None + after = [item.model_dump(mode="json") for item in restored.to_timeline().lanes.turns] + assert after == before + + +def test_output_kind_is_closed_metadata_and_not_answer_content() -> None: + valid = _event( + "llm.first_output", + "llm-main", + "llm", + "reasoning", + 10, + turn_alias="turn", + request_alias="request", + response_alias="response", + attributes={"output_kind": "tool_call"}, + ) + assert valid.attributes == {"output_kind": "tool_call"} + try: + _event( + "llm.first_output", + "llm-main", + "llm", + "invalid", + 10, + turn_alias="turn", + request_alias="request", + response_alias="response", + attributes={"output_kind": "private words"}, + ) + except ValueError as exc: + assert "output_kind" in str(exc) + else: + raise AssertionError("invalid output_kind was accepted") diff --git a/tests/test_postgres_integration.py b/tests/test_postgres_integration.py index 7f39553..680982f 100644 --- a/tests/test_postgres_integration.py +++ b/tests/test_postgres_integration.py @@ -18,8 +18,14 @@ PostgresRunJobQueue, _claim_statement, ) -from voxbench.control_plane.run_api import PostgresRunRepository, StoredRun +from voxbench.control_plane.run_api import ( + PostgresRunRepository, + StoredRun, + _service_event_artifact, +) from voxbench.engine_harness.models import SpanArtifact, TimelineEventArtifact +from voxbench.observability import ServiceEvent +from voxbench.registry.service import RegistryService TEST_POSTGRES_URL_ENV = "VOXBENCH_TEST_POSTGRES_URL" @@ -219,6 +225,106 @@ def test_postgres_round_trips_rtp_packet_gap_evidence( assert incident.direction == "received" +def test_postgres_restart_reconstructs_cascade_causal_analysis( + postgres_runtime: PostgresRuntime, +) -> None: + root = Path(__file__).resolve().parents[1] + resolved = RegistryService.from_files( + config_paths=[root / "examples/v2/configs/cascade.json"], + manifest_paths=(root / "examples/v2/manifests").glob("*.json"), + ).resolve_config("example-cascade") + base = datetime.now(UTC) + run = StoredRun( + run_id=str(uuid4()), + config_hash=resolved.hash, + call_id=None, + conversation_id="", + provider="cascade", + engine="example-telephony", + status="complete", + started_at=base - timedelta(seconds=1), + ended_at=base + timedelta(seconds=1), + resolved_config=resolved.resolved, + recordings=[], + spans=[], + metrics=[], + ) + + def observed( + kind: str, + component_id: str, + role: str, + alias: str, + offset_ms: int, + **relations, + ) -> TimelineEventArtifact: + return _service_event_artifact( + ServiceEvent( + collector_alias="postgres-cascade", + event_alias=alias, + kind=kind, + component_id=component_id, + role=role, + ts=base + timedelta(milliseconds=offset_ms), + clock_domain="application-monotonic", + alignment_uncertainty_ms=1, + **relations, + ) + ) + + run.timeline_events = [ + observed( + "turn.speech_ended", "application", "coordinator", "speech-end", 0, + turn_alias="turn", request_alias="stt", + ), + observed( + "stt.final_emitted", "stt-main", "stt", "stt-final", 20, + session_alias="session", request_alias="stt", + ), + observed( + "turn.committed", "application", "coordinator", "turn", 30, + turn_alias="turn", parent_request_alias="stt", + ), + observed( + "llm.request_started", "llm-main", "llm", "llm-start", 40, + turn_alias="turn", request_alias="llm", parent_request_alias="stt", + response_alias="response", + ), + observed( + "llm.first_answer_text", "llm-main", "llm", "llm-answer", 70, + turn_alias="turn", request_alias="llm", response_alias="response", + ), + observed( + "aggregation.segment_ready", "aggregation-main", "aggregation", "ready", 80, + request_alias="aggregation", parent_request_alias="llm", + response_alias="response", segment_alias="segment", + ), + observed( + "tts.request_started", "tts-main", "tts", "tts-start", 90, + request_alias="tts", parent_request_alias="aggregation", + response_alias="response", segment_alias="segment", + ), + observed( + "tts.first_pcm", "tts-main", "tts", "tts-pcm", 110, + request_alias="tts", response_alias="response", segment_alias="segment", + ), + observed( + "playback.write_started", "output-resampler", "playback", "write", 130, + parent_request_alias="tts", response_alias="response", segment_alias="segment", + ), + ] + expected = run.to_timeline().lanes.turns + postgres_runtime.repository.save(run) + + restored = PostgresRunRepository(postgres_runtime.sessions).get(run.run_id) + assert restored is not None + assert restored.to_timeline().lanes.turns == expected + assert restored.to_timeline().ai_mode == "cascade" + assert [item.role for item in restored.to_timeline().ai_components] == [ + "stt", "llm", "aggregation", "tts", + ] + + def test_postgres_skip_locked_claims_next_job_without_waiting( postgres_runtime: PostgresRuntime, ) -> None: diff --git a/tests/test_registry_v2.py b/tests/test_registry_v2.py new file mode 100644 index 0000000..7240e82 --- /dev/null +++ b/tests/test_registry_v2.py @@ -0,0 +1,500 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from jsonschema import Draft202012Validator +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from voxbench.control_plane.app import create_app +from voxbench.control_plane.models import Base +from voxbench.control_plane.run_api import PostgresRunRepository, StoredRun +from voxbench.engine_harness.harness import EngineHarness +from voxbench.engine_harness.plan import build_stage_plan +from voxbench.engine_harness.storage import LocalRecordingSink +from voxbench.registry.config_views import ai_mode, pipeline_chains, service_components +from voxbench.registry.errors import ConfigValidationError, ManifestNotFoundError +from voxbench.registry.hashing import canonical_json, resolved_hash +from voxbench.registry.service import RegistryService, load_json +from voxbench.schemas_export import export_schemas +from voxbench.synthetic_caller import SyntheticAudioSpec +from voxbench.synthetic_caller.offline import generate_synthetic_artifacts +from voxbench.verification import verify_recordings + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLES = ROOT / "examples/v2" + + +def config_for(mode: str = "cascade") -> dict: + return load_json(EXAMPLES / f"configs/{mode}.json") + + +def manifests_for() -> list[dict]: + return [load_json(path) for path in sorted((EXAMPLES / "manifests").glob("*.json"))] + + +def registry_for(config: dict, manifests: list[dict] | None = None) -> RegistryService: + registry = RegistryService() + for manifest in manifests if manifests is not None else manifests_for(): + registry.register_manifest(manifest) + registry.register_config(config) + return registry + + +def resolve(config: dict): + return registry_for(config).resolve_config(config["meta"]["name"]) + + +def set_path(config: dict, path: str, value: object) -> None: + keys = path.split(".") + target = config + for key in keys[:-1]: + target = target[int(key)] if isinstance(target, list) else target[key] + key = keys[-1] + target[int(key) if isinstance(target, list) else key] = value + + +@pytest.mark.parametrize("mode", ["realtime", "cascade"]) +def test_v2_resolves_independent_chains_and_pinned_services(mode: str) -> None: + config = config_for(mode) + before = deepcopy(config) + result = resolve(config) + roles = {"realtime"} if mode == "realtime" else {"stt", "llm", "tts"} + + assert config == before + assert ai_mode(result.resolved) == mode + assert set(service_components(result.resolved)) == roles + chains = pipeline_chains(result.resolved) + assert chains["input"][0]["id"] == "input-resampler" + assert chains["output"][0]["id"] == "output-resampler" + assert chains["input"][0]["io"]["produces"]["rate"] == 16000 + assert chains["output"][0]["io"]["produces"]["rate"] == 8000 + assert all(stage["type"] == "resampler" for stages in chains.values() for stage in stages) + pins = result.resolved["manifest_pins"] + expected_ids = {"engine", "input-resampler", "output-resampler"} + expected_ids |= {f"{role}-main" for role in roles} + if mode == "cascade": + expected_ids.add("aggregation-main") + assert {pin["component_id"] for pin in pins} == expected_ids + assert all(pin["version"] == "1.0.0" and len(pin["digest"]) == 64 for pin in pins) + assert result.hash == resolve(deepcopy(config)).hash + + +@pytest.mark.parametrize( + ("path", "value", "message"), + [ + ("spec.ai.tts.id", "stt-main", "duplicate or reserved"), + ("spec.media.output_pipeline.0.id", "input-resampler", "duplicate or reserved"), + ("spec.ai.stt.id", "engine", "duplicate or reserved"), + ("spec.ai.text_aggregation.id", "application", "duplicate or reserved"), + ("spec.ai.stt.plugin", "example-tts", "does not support role"), + ("spec.ai.text_aggregation.plugin", "example-input-resampler", "text aggregation"), + ("spec.media.input_pipeline.0.plugin", "example-sentence-aggregator", "requires audio IO"), + ("spec.ai.stt.model", "not-declared", "model is not supported"), + ("spec.ai.stt.input_audio.rate", 44100, "supported audio format"), + ("spec.ai.llm.input_audio", {"encoding": "pcm16", "rate": 8000, "channels": 1}, + "audio on a text boundary"), + ("spec.ai.stt.output_audio", {"encoding": "pcm16", "rate": 8000, "channels": 1}, + "audio on a text boundary"), + ("spec.ai.stt.input_audio.rate", 0, "greater than 0"), + ("spec.ai.stt.input_audio.channels", True, "valid integer"), + ("spec.ai.stt.params", {"unknown": True}, "param_schema"), + ("spec.ai.stt.system_prompt_ref", "unexpected", "Extra inputs"), + ("spec.ai.stt.manifest_version", "9.0.0", "unknown v2 provider manifest"), + ("spec.transport.codec", "unsupported", "engine decode contract"), + ("spec.transport.ptime_ms", 0, "ptime_ms must be positive"), + ("spec.media.input_pipeline", [], "input pipeline boundary format mismatch"), + ("spec.media.output_pipeline", [], "output pipeline boundary format mismatch"), + ("spec.media.input_pipeline.0.plugin", "example-output-resampler", "input pipeline io"), + ("spec.media.output_pipeline.0.plugin", "example-input-resampler", "output pipeline io"), + ("spec.media.input_pipeline.0.host_capabilities", [], "requires host capabilities"), + ("spec.media.input_pipeline.0.requires_host_capability", [], "cannot remove host"), + ("spec.media.input_pipeline.0.invariants_enforced", ["level_preserving"], "cannot invent"), + ("spec.media.input_pipeline.0.overrides", [{"target": "undeclared", "value": 1}], + "override target is not allowed"), + ("spec.media.input_pipeline.0.overrides", [{"target": "quality", "value": 1}], + "invalid type"), + ("spec.media.input_pipeline.0.overrides", [ + {"target": "quality", "value": "high"}, {"target": "quality", "value": "low"}, + ], "duplicate override targets"), + ("spec.turn_taking.speech_detector", "tts-main", "turn authority"), + ("spec.turn_taking.end_of_turn", "llm-main", "turn authority"), + ("spec.turn_taking.interruption", "missing", "turn authority"), + ("spec.turn_taking.end_of_turn", ["application", "stt-main"], "valid string"), + ("spec.turn_taking.owner", "server_vad", "Extra inputs"), + ("spec.ai.provider", "legacy-provider", "Extra inputs"), + ], +) +def test_invalid_v2_configs_fail(path: str, value: object, message: str) -> None: + config = config_for() + set_path(config, path, value) + with pytest.raises((ConfigValidationError, ManifestNotFoundError), match=message): + resolve(config) + + +def test_audio_stage_cannot_replace_pinned_manifest_io() -> None: + config = config_for() + config["spec"]["media"]["input_pipeline"][0]["io"] = { + "mode": "rate_changing", + "accepts": {"encoding": "pcm16", "rate": 8000, "channels": 1}, + "produces": {"encoding": "pcm16", "rate": 44100, "channels": 1}, + } + with pytest.raises(ConfigValidationError, match="cannot replace manifest IO"): + resolve(config) + + +def test_empty_chains_are_valid_only_for_equal_service_engine_formats() -> None: + config = config_for() + manifests = manifests_for() + for manifest in manifests: + if manifest["name"] == "example-telephony": + caps = manifest["engine_caps"] + caps["decoded_input"]["rate"] = 16000 + caps["decoded_output"]["rate"] = 24000 + config["spec"]["media"] = {"input_pipeline": [], "output_pipeline": []} + registry = registry_for(config, manifests) + assert pipeline_chains(registry.resolve_config("example-cascade").resolved) == { + "input": [], "output": [], + } + + +@pytest.mark.parametrize("mode", ["cascade", "realtime"]) +def test_declared_service_can_own_separate_turn_decisions(mode: str) -> None: + config = config_for(mode) + authority = "stt-main" if mode == "cascade" else "realtime-main" + config["spec"]["turn_taking"] = { + "speech_detector": authority, "end_of_turn": authority, "interruption": authority, + } + assert resolve(config).resolved["spec"]["turn_taking"]["end_of_turn"] == authority + + +def test_engine_authority_requires_engine_capability_declaration() -> None: + config = config_for() + config["spec"]["turn_taking"]["interruption"] = "engine" + with pytest.raises(ConfigValidationError, match="turn authority"): + resolve(config) + manifests = manifests_for() + next(m for m in manifests if m["kind"] == "engine")["engine_caps"]["authorities"] = [ + "interruption" + ] + assert registry_for(config, manifests).resolve_config("example-cascade") + + +@pytest.mark.parametrize( + ("path", "value", "message"), + [ + ("service_caps.output_modality", "audio", "modality mismatch"), + ("service_caps.audio_inputs", [], "audio boundaries require formats"), + ("service_caps.audio_outputs", [{"encoding": "pcm16", "rate": 8000, "channels": 1}], + "text boundaries forbid them"), + ("service_caps.roles", ["stt", "stt"], "roles must be unique"), + ("provider_caps", {}, "Extra inputs"), + ("kind", "engine", "engine manifest"), + ], +) +def test_malformed_service_manifests_fail(path: str, value: object, message: str) -> None: + manifest = load_json(EXAMPLES / "manifests/stt.json") + set_path(manifest, path, value) + with pytest.raises(ConfigValidationError, match=message): + RegistryService().register_manifest(manifest) + + +def test_text_aggregation_cannot_claim_audio_invariants() -> None: + manifest = load_json(EXAMPLES / "manifests/aggregation.json") + manifest["invariants_applicable"] = ["duration_preserving"] + with pytest.raises(ConfigValidationError, match="text processors cannot declare audio"): + RegistryService().register_manifest(manifest) + + +def test_tts_manifest_cannot_claim_caller_authority() -> None: + manifest = load_json(EXAMPLES / "manifests/tts.json") + manifest["service_caps"]["authorities"] = ["speech_detector"] + with pytest.raises(ConfigValidationError, match="TTS cannot own"): + RegistryService().register_manifest(manifest) + + +def test_invalid_plugin_parameter_schema_is_a_registry_error() -> None: + manifest = load_json(EXAMPLES / "manifests/stt.json") + manifest["param_schema"] = {"type": "invalid-json-schema-type"} + with pytest.raises(ConfigValidationError): + RegistryService().register_manifest(manifest) + + +def test_ambiguous_allowed_overrides_are_rejected() -> None: + manifest = load_json(EXAMPLES / "manifests/input-resampler.json") + manifest["allowed_overrides"].append({"target": "quality", "type": "integer"}) + with pytest.raises(ConfigValidationError, match="override targets must be unique"): + RegistryService().register_manifest(manifest) + + +def test_manifest_identity_is_immutable_but_multiple_versions_are_selectable() -> None: + config = config_for() + registry = registry_for(config) + original = registry.resolve_config("example-cascade") + manifest = load_json(EXAMPLES / "manifests/stt.json") + assert registry.register_manifest(deepcopy(manifest)) + manifest["service_caps"]["lifecycle"]["usage"] = True + with pytest.raises(ConfigValidationError, match="identity is immutable"): + registry.register_manifest(manifest) + manifest["version"] = "1.1.0" + registry.register_manifest(manifest) + assert registry.resolve_config("example-cascade").hash == original.hash + config["spec"]["ai"]["stt"]["manifest_version"] = "1.1.0" + registry.register_config(config) + updated = registry.resolve_config("example-cascade") + assert updated.hash != original.hash + pin = next(p for p in updated.resolved["manifest_pins"] if p["component_id"] == "stt-main") + assert pin["version"] == "1.1.0" + assert pin["digest"] == resolved_hash(registry.register_manifest(manifest).model_dump( + mode="json", exclude_none=True, + )) + + +def test_mutating_registration_result_does_not_change_pinned_manifest() -> None: + config = config_for() + registry = registry_for(config) + original = registry.resolve_config("example-cascade") + manifest = registry.register_manifest(load_json(EXAMPLES / "manifests/stt.json")) + manifest.service_caps.lifecycle.usage = True + assert registry.resolve_config("example-cascade").hash == original.hash + + +def test_registered_v2_config_is_a_snapshot_of_the_supplied_object() -> None: + config = config_for() + registry = registry_for(config) + original = registry.resolve_config("example-cascade") + config["spec"]["ai"]["stt"]["model"] = "unregistered-change" + assert registry.resolve_config("example-cascade").hash == original.hash + + +def test_v1_and_v2_provider_names_coexist_without_changing_legacy_resolution() -> None: + registry = RegistryService.from_files( + config_paths=[ROOT / "examples/configs/valid-baseline.json"], + manifest_paths=(ROOT / "examples/manifests").rglob("*.json"), + ) + original = registry.resolve_config("baseline") + config = config_for() + config["spec"]["ai"]["stt"]["plugin"] = "gemini" + for manifest in manifests_for(): + if manifest["name"] == "example-stt": + manifest["name"] = "gemini" + registry.register_manifest(manifest) + registry.register_config(config) + assert registry.resolve_config("example-cascade").resolved["spec"]["ai"]["stt"]["plugin"] == ( + "gemini" + ) + assert registry.resolve_config("baseline") == original + + +@pytest.mark.parametrize("role", ["stt", "llm", "tts"]) +def test_same_mode_overlay_changes_only_selected_model(role: str) -> None: + config = config_for() + manifests = manifests_for() + manifest = next(m for m in manifests if m["name"] == f"example-{role}") + manifest["service_caps"]["supported_models"].append("pinned-alternative-model") + registry = registry_for(config, manifests) + base = registry.resolve_config("example-cascade") + child = { + "apiVersion": "voxbench/v2", "kind": "VoiceConfig", + "meta": {"name": "child", "version": "1.1.0", "parent": "example-cascade"}, + "spec": {"ai": {role: {"model": "pinned-alternative-model"}}}, + } + registry.register_config(child) + result = registry.resolve_config("child") + assert result.resolved["spec"]["ai"][role]["model"] == "pinned-alternative-model" + for other in {"stt", "llm", "tts"} - {role}: + assert result.resolved["spec"]["ai"][other] == base.resolved["spec"]["ai"][other] + assert result.hash != base.hash + assert registry.resolve_config("child").hash == result.hash + + +@pytest.mark.parametrize( + ("base_version", "child_version", "child_mode", "message"), + [ + ("voxbench/v2", "voxbench/v1", None, "cross-version"), + ("voxbench/v1", "voxbench/v2", "cascade", "cross-version"), + ("voxbench/v2", "voxbench/v2", "realtime", "cross-mode"), + ], +) +def test_overlay_cannot_cross_version_or_mode( + base_version: str, child_version: str, child_mode: str | None, message: str, +) -> None: + config = config_for() + config["apiVersion"] = base_version + registry = registry_for(config) + child = { + "apiVersion": child_version, "meta": {"name": "child", "parent": "example-cascade"}, + "spec": {"ai": {"mode": child_mode}} if child_mode else {}, + } + registry.register_config(child) + with pytest.raises(ConfigValidationError, match=message): + registry.resolve_config("child") + + +@pytest.mark.parametrize("spec", [None, [], {"ai": None}, {"ai": []}]) +def test_malformed_overlay_shapes_fail_as_registry_errors(spec: object) -> None: + registry = registry_for(config_for()) + registry.register_config({ + "apiVersion": "voxbench/v2", "kind": "VoiceConfig", + "meta": {"name": "child", "version": "1.1.0", "parent": "example-cascade"}, + "spec": spec, + }) + with pytest.raises(ConfigValidationError, match="v2 overlay"): + registry.resolve_config("child") + + +def test_exported_v1_schemas_stay_identical_and_v2_examples_validate(tmp_path: Path) -> None: + export_schemas(tmp_path) + for filename in ("config.schema.json", "manifest.schema.json"): + assert (tmp_path / filename).read_bytes() == (ROOT / "schemas" / filename).read_bytes() + config_schema = load_json(tmp_path / "config.v2.schema.json") + manifest_schema = load_json(tmp_path / "manifest.v2.schema.json") + for schema in (config_schema, manifest_schema): + Draft202012Validator.check_schema(schema) + for mode in ("cascade", "realtime"): + Draft202012Validator(config_schema).validate(config_for(mode)) + for manifest in manifests_for(): + Draft202012Validator(manifest_schema).validate(manifest) + + +def test_v2_core_resolution_does_not_import_framework_sdks() -> None: + code = """ +import importlib.abc, sys +class BlockFrameworks(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if fullname.split('.')[0] in {'pipecat', 'livekit'}: + raise AssertionError('core imported middleware: ' + fullname) +sys.meta_path.insert(0, BlockFrameworks()) +from pathlib import Path +from voxbench.registry.service import RegistryService +for mode in ('cascade', 'realtime'): + r = RegistryService.from_files( + config_paths=[f'examples/v2/configs/{mode}.json'], + manifest_paths=Path('examples/v2/manifests').glob('*.json'), + ).resolve_config(f'example-{mode}') + assert len(r.hash) == 64 +""" + result = subprocess.run([sys.executable, "-c", code], cwd=ROOT, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("mode", ["cascade", "realtime"]) +@pytest.mark.parametrize("endpoint", ["/runs", "/runs/async"]) +def test_v2_run_api_fails_explicitly_until_observation_execution_support( + mode: str, endpoint: str, tmp_path: Path, +) -> None: + config = config_for(mode) + artifact_root = tmp_path / "recordings" + client = TestClient(create_app(artifact_root=artifact_root)) + response = client.post(endpoint, json={ + "config_name": config["meta"]["name"], "configs": [config], "manifests": manifests_for(), + }) + assert response.status_code == 400 + assert "configuration resolution only" in response.json()["detail"] + assert client.get("/runs").json() == [] + assert not artifact_root.exists() + + +@pytest.mark.parametrize("mode", ["cascade", "realtime"]) +def test_v2_observation_run_can_start_without_executing_audio( + mode: str, tmp_path: Path, +) -> None: + config = config_for(mode) + artifact_root = tmp_path / "recordings" + response = TestClient(create_app(artifact_root=artifact_root)).post( + "/runs/observed", + json={ + "config_name": config["meta"]["name"], + "configs": [config], + "manifests": manifests_for(), + }, + ) + assert response.status_code == 200 + assert response.json()["status"] == "running" + assert not artifact_root.exists() + + +def test_v2_legacy_audio_tools_fail_before_writing_artifacts(tmp_path: Path) -> None: + config = resolve(config_for()).resolved + root = tmp_path / "artifacts" + with pytest.raises(ConfigValidationError, match="stage planning"): + build_stage_plan(config) + harness = EngineHarness(recording_sink=LocalRecordingSink(root)) + with pytest.raises(ConfigValidationError, match="legacy harness execution"): + harness.run_once(run_id="test", resolved_config=config, config_hash=resolved_hash(config)) + with pytest.raises(ConfigValidationError, match="synthetic audio generation"): + generate_synthetic_artifacts( + resolved_config=config, output_root=root, + audio_spec=SyntheticAudioSpec(8000, 1, 0.1, 1000, 440), + ) + with pytest.raises(ConfigValidationError, match="recording verification"): + verify_recordings(resolved_config=config, recordings=[]) + assert not root.exists() + + +@pytest.mark.parametrize("overlay", [False, True]) +def test_v1_resolved_bytes_hash_and_read_views_remain_unchanged(overlay: bool) -> None: + fixture = "v1-overlay" if overlay else "v1-baseline" + golden = (ROOT / f"tests/fixtures/configs/{fixture}.resolved.json").read_text().strip() + registry = RegistryService.from_files( + config_paths=[ROOT / "examples/configs/valid-baseline.json"], + manifest_paths=(ROOT / "examples/manifests").rglob("*.json"), + ) + if overlay: + registry.register_config({ + "apiVersion": "voxbench/v1", "kind": "VoiceConfig", + "meta": {"name": "overlay-golden", "version": "1.0.1", "parent": "baseline"}, + "spec": {"ai": {"model": "pinned-overlay-model"}}, + }) + result = registry.resolve_config("overlay-golden" if overlay else "baseline") + assert canonical_json(result.resolved) == golden + expected_hash = ( + "6e0d5937a5fcf3e94cc3581533246ecee81e1b2c4528d896ca830173f176bdfa" + if overlay else "6b3467de65af01c527f2dfd95e260fe734e357f8acc392916b1d86567199bde1" + ) + assert result.hash == expected_hash + assert ai_mode(result.resolved) == "realtime" + assert service_components(result.resolved)["realtime"]["plugin"] == "gemini" + assert list(pipeline_chains(result.resolved)) == ["legacy"] + service_components(result.resolved)["realtime"]["params"]["example"] = "view-only" + pipeline_chains(result.resolved)["legacy"][0]["params"]["input_rate"] = 1234 + assert canonical_json(result.resolved) == golden + + +def test_v2_read_views_cannot_mutate_the_resolved_config() -> None: + config = resolve(config_for()).resolved + before = canonical_json(config) + service_components(config)["stt"]["input_audio"]["rate"] = 1234 + pipeline_chains(config)["input"][0]["io"]["accepts"]["rate"] = 1234 + assert canonical_json(config) == before + + +def test_stored_v1_config_loads_without_v2_defaults() -> None: + engine = create_engine("sqlite+pysqlite://") + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + repository = PostgresRunRepository(sessions) + golden = (ROOT / "tests/fixtures/configs/v1-baseline.resolved.json").read_text().strip() + config = json.loads(golden) + run = StoredRun( + run_id=str(uuid4()), config_hash=resolved_hash(config), call_id=None, + conversation_id="historical-v1", provider="gemini", engine="asterisk", + status="completed", started_at=datetime.now(UTC), ended_at=datetime.now(UTC), + resolved_config=config, recordings=[], spans=[], metrics=[], + ) + repository.save(run) + restored = PostgresRunRepository(sessions).get(run.run_id) + assert restored is not None + assert canonical_json(restored.resolved_config) == golden + assert restored.config_hash == run.config_hash + assert ai_mode(restored.resolved_config) == "realtime" + assert canonical_json(restored.resolved_config) == golden + engine.dispose() diff --git a/tests/test_service_observations.py b/tests/test_service_observations.py new file mode 100644 index 0000000..a338a06 --- /dev/null +++ b/tests/test_service_observations.py @@ -0,0 +1,618 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from jsonschema import Draft202012Validator +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +import voxbench.control_plane.run_api as run_api_module +from voxbench.control_plane.app import create_app +from voxbench.control_plane.models import Base +from voxbench.control_plane.run_api import PostgresRunRepository +from voxbench.observability import ObservationBatch, ServiceEvent, VoxBenchObserver +from voxbench.registry.service import load_json +from voxbench.schemas_export import export_schemas + +ROOT = Path(__file__).resolve().parents[1] +V2 = ROOT / "examples/v2" + + +def _v2_payload(mode: str = "cascade") -> dict: + config = load_json(V2 / f"configs/{mode}.json") + return { + "config_name": config["meta"]["name"], + "configs": [config], + "manifests": [load_json(path) for path in sorted((V2 / "manifests").glob("*.json"))], + "call_id": f"service-observation-{mode}", + } + + +class ApiTransport: + def __init__(self, client: TestClient) -> None: + self.client = client + + def send(self, batch: ObservationBatch) -> None: + response = self.client.post("/v1/observations", json=batch.to_payload()) + response.raise_for_status() + + +class RecordingTransport: + def __init__(self) -> None: + self.batches: list[ObservationBatch] = [] + + def send(self, batch: ObservationBatch) -> None: + self.batches.append(batch) + + +def _start(client: TestClient, mode: str = "cascade") -> str: + response = client.post("/runs/observed", json=_v2_payload(mode)) + assert response.status_code == 200, response.text + return response.json()["run_id"] + + +def _event(kind: str, component_id: str, role: str, alias: str, **relations) -> ServiceEvent: + return ServiceEvent( + collector_alias="application-a", + event_alias=alias, + kind=kind, + component_id=component_id, + role=role, + ts=datetime(2026, 9, 27, 1, 2, tzinfo=UTC) + timedelta(milliseconds=len(alias)), + **relations, + ) + + +def _turn_events(prefix: str, *, segment_count: int = 2) -> list[ServiceEvent]: + turn = f"turn-{prefix}" + stt_request = f"stt-request-{prefix}" + response = f"response-{prefix}" + llm_request = f"llm-request-{prefix}" + events = [ + _event( + "stt.final_emitted", "stt-main", "stt", f"{prefix}-stt-final", + session_alias="stt-session", request_alias=stt_request, + segment_alias=f"stt-segment-{prefix}", + ), + _event( + "turn.committed", "application", "coordinator", f"{prefix}-turn", + turn_alias=turn, parent_request_alias=stt_request, + ), + _event( + "llm.request_started", "llm-main", "llm", f"{prefix}-llm-start", + turn_alias=turn, request_alias=llm_request, + parent_request_alias=stt_request, response_alias=response, + ), + _event( + "llm.first_answer_text", "llm-main", "llm", f"{prefix}-llm-first", + turn_alias=turn, request_alias=llm_request, response_alias=response, + ), + ] + for ordinal in range(segment_count): + segment = f"segment-{prefix}-{ordinal}" + aggregation_request = f"aggregation-request-{prefix}-{ordinal}" + tts_request = f"tts-request-{prefix}-{ordinal}" + events.extend( + [ + _event( + "aggregation.segment_ready", "aggregation-main", "aggregation", + f"{prefix}-aggregation-{ordinal}", request_alias=aggregation_request, + parent_request_alias=llm_request, response_alias=response, + segment_alias=segment, attributes={"sequence_ordinal": ordinal}, + ), + _event( + "tts.request_started", "tts-main", "tts", f"{prefix}-tts-start-{ordinal}", + request_alias=tts_request, parent_request_alias=aggregation_request, + response_alias=response, segment_alias=segment, + ), + _event( + "tts.first_pcm", "tts-main", "tts", f"{prefix}-tts-first-{ordinal}", + request_alias=tts_request, response_alias=response, segment_alias=segment, + attributes={"observed_boundary": "first-pcm-frame"}, + ), + _event( + "tts.response_completed", "tts-main", "tts", + f"{prefix}-tts-complete-{ordinal}", request_alias=tts_request, + response_alias=response, segment_alias=segment, + attributes={"terminal_outcome": "completed"}, + ), + _event( + "playback.write_started", "output-resampler", "playback", + f"{prefix}-playback-{ordinal}", parent_request_alias=tts_request, + response_alias=response, segment_alias=segment, + attributes={"observed_boundary": "local-write"}, + ), + ] + ) + events.append( + _event( + "llm.response_completed", "llm-main", "llm", f"{prefix}-llm-complete", + turn_alias=turn, request_alias=llm_request, response_alias=response, + attributes={"terminal_outcome": "completed"}, + ) + ) + return events + + +def test_fake_multi_turn_multi_segment_cascade_ingests_without_content(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path / "audio")) + run_id = _start(client) + observer = VoxBenchObserver(run_id, ApiTransport(client)) + observer.observe_service_event( + _event( + "service.session_started", "stt-main", "stt", "stt-session-start", + session_alias="stt-session", + ) + ) + events = _turn_events("one") + _turn_events("two") + for event in events: + observer.observe_service_event(event) + assert observer.flush() == len(events) + 1 + assert observer.pending_count == 0 + + completed = client.post(f"/runs/{run_id}/complete", json={}) + assert completed.status_code == 200 + assert completed.json()["recordings"] == [] + timeline = client.get(f"/runs/{run_id}/timeline").json() + service_events = [ + event for event in timeline["lanes"]["events"] + if event["source"] == "service_observation" + ] + assert len(service_events) == len(events) + 1 + assert sum(event["name"] == "tts.first_pcm" for event in service_events) == 4 + assert sum(event["name"] == "turn.committed" for event in service_events) == 2 + assert all(len(event["attributes"]) <= 16 for event in service_events) + encoded = json.dumps(service_events).lower() + for forbidden in ("transcript", "prompt", "generated_text", "content_hash", "provider_id"): + assert forbidden not in encoded + first_pcm = next(event for event in service_events if event["name"] == "tts.first_pcm") + assert first_pcm["attributes"]["observed_boundary"] == "first-pcm-frame" + assert "local-write" not in first_pcm["attributes"].values() + playback = next(event for event in service_events if event["name"] == "playback.write_started") + assert playback["attributes"]["observed_boundary"] == "local-write" + + +@pytest.mark.parametrize( + "injected", + [ + {"transcript": "private words"}, + {"prompt": "private words"}, + {"generated_text": "private words"}, + {"content_hash": "abc"}, + {"provider_id": "raw-123"}, + {"url": "https://provider.example/item"}, + {"api_key": "secret"}, + {"raw_payload": {}}, + {"run_id": "other-run"}, + ], +) +def test_service_event_contract_rejects_content_raw_ids_and_cross_run_fields( + injected: dict, +) -> None: + payload = _event( + "stt.final_emitted", "stt-main", "stt", "strict", + session_alias="session", request_alias="request", + ).to_payload() + payload.update(injected) + with pytest.raises(ValueError): + ServiceEvent.model_validate(payload) + + +def test_service_attributes_are_allowlisted_scalar_safe_and_bounded() -> None: + base = _event( + "stt.final_emitted", "stt-main", "stt", "attrs", + session_alias="session", request_alias="request", + ).to_payload() + for attributes in ( + {"unknown": 1}, + {"failure_alias": "https://provider.example/error"}, + {"duration_ms": float("inf")}, + {"duration_ms": -1}, + {"input_units": True}, + {"retry": 1}, + {"failure_alias": None}, + {"terminal_outcome": {"nested": "payload"}}, + ): + with pytest.raises(ValueError): + ServiceEvent.model_validate({**base, "attributes": attributes}) + too_many = { + "attempt_ordinal": 1, "sequence_ordinal": 1, "duration_ms": 1, + "audio_duration_ms": 1, "input_units": 1, "output_units": 1, + "cached_input_units": 1, "tool_call_count": 1, "queue_depth": 1, + "sample_rate_hz": 1, "channels": 1, "encoding": "pcm16", + } + with pytest.raises(ValueError, match="at most 16"): + ServiceEvent.model_validate({**base, "attributes": too_many}) + + +def test_service_timestamp_must_include_offset_and_is_normalized_to_utc() -> None: + payload = _event( + "stt.final_emitted", "stt-main", "stt", "timestamp", + session_alias="session", request_alias="request", + ).to_payload() + with pytest.raises(ValueError, match="UTC offset"): + ServiceEvent.model_validate({**payload, "ts": "2026-09-27T10:00:00"}) + normalized = ServiceEvent.model_validate({**payload, "ts": "2026-09-27T10:00:00+09:00"}) + assert normalized.ts.isoformat() == "2026-09-27T01:00:00+00:00" + + +def test_component_role_and_turn_authority_are_checked_against_run_config(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + wrong_role = _event( + "tts.first_pcm", "stt-main", "tts", "wrong-role", + request_alias="request", response_alias="response", segment_alias="segment", + ) + response = client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [wrong_role.to_payload()]}, + ) + assert response.status_code == 400 + assert "not configured for service role" in response.json()["detail"] + + wrong_authority = _event( + "turn.committed", "engine", "coordinator", "wrong-authority", + turn_alias="turn", parent_request_alias="stt-request", + ) + response = client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [wrong_authority.to_payload()]}, + ) + assert response.status_code == 400 + assert "end_of_turn authority" in response.json()["detail"] + + +def test_retry_is_idempotent_changed_payload_conflicts_and_collectors_do_not_collide( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + event = _event( + "stt.final_emitted", "stt-main", "stt", "same-event", + session_alias="session", request_alias="request", + attributes={"sequence_ordinal": 1}, + ) + payload = {"run_id": run_id, "service_events": [event.to_payload()]} + assert client.post("/v1/observations", json=payload).status_code == 200 + assert client.post("/v1/observations", json=payload).status_code == 200 + + conflict = event.model_copy(update={"attributes": {"sequence_ordinal": 2}}) + response = client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [conflict.to_payload()]}, + ) + assert response.status_code == 409 + + other_collector = event.model_copy(update={"collector_alias": "application-b"}) + assert client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [other_collector.to_payload()]}, + ).status_code == 200 + timeline = client.get(f"/runs/{run_id}/timeline").json() + matching = [ + event + for event in timeline["lanes"]["events"] + if event["name"] == "stt.final_emitted" + ] + assert {event["event_id"] for event in matching} == { + "service:application-a:same-event", "service:application-b:same-event", + } + + +def test_unknown_parent_coverage_resolves_when_parent_arrives_out_of_order(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + child = _event( + "aggregation.segment_ready", "aggregation-main", "aggregation", "child", + request_alias="aggregation-request", parent_request_alias="late-request", + response_alias="response", segment_alias="segment", + ) + assert client.post( + "/v1/observations", json={"run_id": run_id, "service_events": [child.to_payload()]}, + ).status_code == 200 + child_event = next( + event for event in client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + if event["event_id"].endswith(":child") + ) + assert child_event["attributes"]["unresolved_relations"] == "parent_request" + + parent = _event( + "llm.request_started", "llm-main", "llm", "late-parent", + turn_alias="turn", request_alias="late-request", response_alias="response", + unobserved_relations=["parent_request"], + ) + assert client.post( + "/v1/observations", json={"run_id": run_id, "service_events": [parent.to_payload()]}, + ).status_code == 200 + child_event = next( + event for event in client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + if event["event_id"].endswith(":child") + ) + assert "unresolved_relations" not in child_event["attributes"] + parent_event = next( + event for event in client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + if event["event_id"].endswith(":late-parent") + ) + assert parent_event["attributes"]["unobserved_relations"] == "parent_request" + + +def test_cancellation_requested_acknowledged_and_missing_terminal_remain_distinct( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + requested = _event( + "cancel.requested", "tts-main", "tts", "cancel-request", + request_alias="tts-request", generation_epoch=2, + attributes={"cancel_scope": "current-generation"}, + ) + started_without_terminal = _event( + "tts.request_started", "tts-main", "tts", "unfinished", + request_alias="unfinished-request", parent_request_alias="aggregation-request", + response_alias="response", segment_alias="segment", + ) + assert client.post( + "/v1/observations", + json={ + "run_id": run_id, + "service_events": [requested.to_payload(), started_without_terminal.to_payload()], + }, + ).status_code == 200 + names = [ + event["name"] for event in client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + if event["source"] == "service_observation" + ] + assert "cancel.requested" in names + assert "cancel.acknowledged" not in names + assert "tts.response_completed" not in names + + acknowledged = _event( + "cancel.acknowledged", "tts-main", "tts", "cancel-ack", + request_alias="tts-request", generation_epoch=2, + attributes={"terminal_outcome": "cancelled"}, + ) + assert client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [acknowledged.to_payload()]}, + ).status_code == 200 + + +def test_generic_path_cannot_bypass_reserved_service_event_validation(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + response = client.post( + "/v1/observations", + json={ + "run_id": run_id, + "timeline_events": [{ + "event_id": "bypass", "category": "provider", "name": "stt.final_emitted", + "source": "custom", "attributes": {"transcript": "private"}, + }], + }, + ) + assert response.status_code == 422 + for key, value in ( + ("event_id", "service:collector:event"), + ("source", "service_observation"), + ): + generic = { + "event_id": "generic", "category": "provider", "name": "custom", + "source": "custom", key: value, + } + response = client.post( + "/v1/observations", + json={"run_id": run_id, "timeline_events": [generic]}, + ) + assert response.status_code == 422 + + +def test_generic_timeline_retries_are_idempotent_and_changed_payload_conflicts( + tmp_path: Path, +) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + event = { + "event_id": "generic-retry", "category": "provider", "name": "custom.boundary", + "source": "custom", "ts": "2026-09-27T01:00:00+00:00", + } + payload = {"run_id": run_id, "timeline_events": [event]} + assert client.post("/v1/observations", json=payload).status_code == 200 + assert client.post("/v1/observations", json=payload).status_code == 200 + conflict = {**event, "name": "custom.changed"} + response = client.post( + "/v1/observations", json={"run_id": run_id, "timeline_events": [conflict]}, + ) + assert response.status_code == 409 + timeline = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + assert sum(event["event_id"] == "generic-retry" for event in timeline) == 1 + + +def test_combined_event_batch_limit_and_observer_splitting() -> None: + transport = RecordingTransport() + observer = VoxBenchObserver("run", transport) + for ordinal in range(129): + observer.observe_service_event( + _event( + "stt.final_emitted", "stt-main", "stt", f"event-{ordinal}", + session_alias="session", request_alias=f"request-{ordinal}", + ) + ) + assert observer.flush() == 129 + assert [len(batch.service_events) for batch in transport.batches] == [128, 1] + + payload = { + "run_id": "run", + "timeline_events": [ + {"event_id": f"generic-{n}", "category": "provider", "name": "custom", + "source": "custom"} + for n in range(64) + ], + "service_events": [ + _event( + "stt.final_emitted", "stt-main", "stt", f"combined-{n}", + session_alias="session", request_alias=f"request-{n}", + ).to_payload() + for n in range(65) + ], + } + from voxbench.control_plane.run_api import ObservationBatchRequest + + with pytest.raises(ValueError, match="128 combined"): + ObservationBatchRequest.model_validate(payload) + + +def test_observer_pending_budget_exposes_drops_without_blocking_flush() -> None: + transport = RecordingTransport() + observer = VoxBenchObserver("run", transport, max_pending_items=2) + for ordinal in range(3): + observer.observe_service_event( + _event( + "stt.final_emitted", "stt-main", "stt", f"bounded-{ordinal}", + session_alias="session", request_alias=f"request-{ordinal}", + ) + ) + assert observer.pending_count == 2 + assert observer.observation_drop_count == 1 + assert observer.flush() == 2 + assert observer.observation_drop_count == 1 + + +def test_failed_flush_restores_admitted_event_and_drops_newer_concurrent_arrival() -> None: + class ArrivingFailureTransport: + observer: VoxBenchObserver + + def send(self, _batch: ObservationBatch) -> None: + self.observer.observe_service_event( + _event( + "stt.final_emitted", "stt-main", "stt", "newer", + session_alias="session", request_alias="newer-request", + ) + ) + raise RuntimeError("send failed") + + transport = ArrivingFailureTransport() + observer = VoxBenchObserver("run", transport, max_pending_items=1) + transport.observer = observer + original = _event( + "stt.final_emitted", "stt-main", "stt", "original", + session_alias="session", request_alias="original-request", + ) + observer.observe_service_event(original) + with pytest.raises(RuntimeError, match="send failed"): + observer.flush() + assert observer.pending_count == 1 + assert observer.observation_drop_count == 1 + recording = RecordingTransport() + observer.transport = recording + assert observer.flush() == 1 + assert recording.batches[0].service_events[0].event_alias == "original" + + +def test_per_run_event_budget_rejects_without_partial_mutation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(run_api_module, "MAX_TIMELINE_EVENTS_PER_RUN", 2) + client = TestClient(create_app(artifact_root=tmp_path)) + run_id = _start(client) + first = _turn_events("budget", segment_count=0)[:2] + assert client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [event.to_payload() for event in first]}, + ).status_code == 200 + rejected = _event( + "stt.final_emitted", "stt-main", "stt", "over-budget", + session_alias="session", request_alias="request", + ) + response = client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [rejected.to_payload()]}, + ) + assert response.status_code == 429 + events = client.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + assert all(not event["event_id"].endswith(":over-budget") for event in events) + + +def test_service_events_persist_and_reconstruct_from_sql_repository(tmp_path: Path) -> None: + engine = create_engine( + "sqlite+pysqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + sessions = sessionmaker(bind=engine, expire_on_commit=False) + repository = PostgresRunRepository(sessions) + first = TestClient(create_app(artifact_root=tmp_path, repository=repository)) + run_id = _start(first) + events = _turn_events("persist", segment_count=1) + response = first.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [event.to_payload() for event in events]}, + ) + assert response.status_code == 200 + assert response.json()["service_event_count"] == len(events) + + restarted = TestClient( + create_app( + artifact_root=tmp_path, + repository=PostgresRunRepository(sessions), + ) + ) + persisted = [ + event for event in restarted.get(f"/runs/{run_id}/timeline").json()["lanes"]["events"] + if event["source"] == "service_observation" + ] + assert len(persisted) == len(events) + assert {event["name"] for event in persisted} >= { + "stt.final_emitted", "llm.request_started", "tts.first_pcm", "playback.write_started", + } + engine.dispose() + + +def test_exported_service_event_schema_validates_plain_http_payload(tmp_path: Path) -> None: + export_schemas(tmp_path) + schema = load_json(tmp_path / "service-event.v1.schema.json") + Draft202012Validator.check_schema(schema) + payload = _event( + "stt.final_emitted", "stt-main", "stt", "schema", + session_alias="session", request_alias="request", + ).to_payload() + Draft202012Validator(schema).validate(payload) + assert (tmp_path / "service-event.v1.schema.json").read_bytes() == ( + ROOT / "schemas/service-event.v1.schema.json" + ).read_bytes() + + +def test_v1_realtime_run_can_use_same_service_contract(tmp_path: Path) -> None: + client = TestClient(create_app(artifact_root=tmp_path)) + config = load_json(ROOT / "examples/configs/live-demo-openai-realtime.json") + payload = { + "config_name": config["meta"]["name"], + "configs": [config], + "manifests": [ + load_json(ROOT / relative) + for relative in ( + "examples/manifests/engine/asterisk.json", + "examples/manifests/provider/openai-realtime.json", + "examples/manifests/processor/resampler.json", + "examples/manifests/processor/agc.json", + "examples/manifests/processor/limiter.json", + "examples/manifests/processor/serializer.json", + ) + ], + } + run_id = client.post("/runs/observed", json=payload).json()["run_id"] + event = _event( + "realtime.first_audio", "ai", "realtime", "v1-realtime", + request_alias="request", response_alias="response", + ) + response = client.post( + "/v1/observations", + json={"run_id": run_id, "service_events": [event.to_payload()]}, + ) + assert response.status_code == 200 + assert response.json()["service_event_count"] == 1 From 0c89ecbe87a883649ed6361a84631a3ed253e77e Mon Sep 17 00:00:00 2001 From: engkimo Date: Sun, 27 Sep 2026 15:43:21 +0900 Subject: [PATCH 2/2] docs: record cascade review branch status --- MEMORY.md | 8 ++--- PROGRESS.md | 7 +++++ README.md | 13 ++++---- ROADMAP.md | 7 +++-- docs/cascade-analysis.md | 3 +- docs/cascade-config-v2.md | 5 +-- docs/cascade-design.md | 19 +++++++----- docs/cascade-issue-plan.md | 9 +++--- docs/cascade-observation-api.md | 3 +- docs/implementation-status.md | 54 ++++++++++++++++++--------------- 10 files changed, 75 insertions(+), 53 deletions(-) diff --git a/MEMORY.md b/MEMORY.md index b626b7f..0fa4ce3 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -7,7 +7,7 @@ - 実装状況の入口は [implementation status](docs/implementation-status.md)。コミット済み、検証済みlocal candidate、計画、deployment検証残件を分ける。 - 設計は `DESIGN.md`、[診断エージェント設計](docs/diagnostic-agent-design.md)、[cascade設計案](docs/cascade-design.md)を参照する。設計の存在だけで実装済みとは判定しない。 - `PROGRESS.md`と日付付きSerena実装memoryは当時のsnapshot。古い「未実装」や進捗率を現在の状態として再利用しない。 -- 公開用worktreeのコミット基準は`ae575b6`。UI commandとlocal diagnostic SSEは未コミットのlocal candidateで、必要な新規module/test/Web fileも未追跡である。詳細と独立した安定化手順はstatus文書に記録した。 +- `main`の監査基準は`ae575b6`。Cascade C1–C3Aは`agent/cascade-c1-c3`の[Draft PR #25](https://github.com/engkimo/voxbench/pull/25)にコミット済みで、未merge。UI commandとlocal diagnostic SSEはこのPRに含めておらず、引き続き未コミットのlocal candidateである。詳細と独立した安定化手順はstatus文書に記録した。 - 2026-09-18にlocal candidateを含むPython 3.14.3のsuiteで330 passed / 4 skipped、Ruff、Web build成功を確認。Postgres実接続、browser、production認証、公開CIの検証とは区別する。 - 同日C1実装後の最終検証：現在のPython suiteは408 passed / 4 skipped、Ruff成功。コミット済み基準へC1だけを加えたcheckoutは401 passed / 4 skipped、Ruff成功。78件のC1テストに加え既存テストも成功。後者はUI/SSE candidate不要で、dependencyは既存環境を共有した。 - ユーザーは調査・計画・設計に続いて実装を指示した。candidateを暗黙の公開baselineにせず、cascade契約はコミット済みv1との互換性を基準に実装する。 @@ -22,14 +22,14 @@ - 共通service観測契約とconformance fixtureを中心に置く。framework adapterは任意dependency、直接接続の例はframework不要で動かす。専用adapterのないアプリにもapplication-owned hook/HTTP経路を提供する。 - テスト通話はframework非依存の`RuntimeAdapter`/application launcher契約越しに起動する設計。通話runtimeの管理主体、model service adapter、観測transportを分離する。 - 現在もobserver/HTTP ingestとbuilt-in直接接続Realtime経路はPipecat非依存。Pipecat helperは薄い構築境界で、他framework専用adapter、cascade観測・実行、横断conformanceは未実装。 -- #17/C1の設定契約をlocal candidateとして実装した。`voxbench/v2`のrealtime/cascade union、service role/modality、input/output PCM chain、一意なcomponent ID、判断主体、manifest版/digest固定とoverlay検証に対応する。v1のresolved object/hashは維持する。現在の契約と使い方は[cascade config v2](docs/cascade-config-v2.md)。 +- #17/C1の設定契約をDraft PR #25のreview branch candidateとして実装した。`voxbench/v2`のrealtime/cascade union、service role/modality、input/output PCM chain、一意なcomponent ID、判断主体、manifest版/digest固定とoverlay検証に対応する。v1のresolved object/hashは維持する。現在の契約と使い方は[cascade config v2](docs/cascade-config-v2.md)。 - turn/request/response/TTS segment/epochを明示相関する。並行処理のdurationを足してend-to-end latencyにしない。clockや相関の不足はindeterminateとする。 - 文字列はアプリruntime内で必要だが、VoxBenchのservice観測はmetadata-only。transcript/prompt/LLM文/本文hashを標準保存・診断model送信しない。 - 音声のmedia timeとrun timeをsegment単位で対応付ける。STTとTTS間にaudio duration/level preservationを適用しない。 - C1設定契約だけでは実通話成功を意味しない。C2が観測専用runとservice event ingestを追加した後も、executionとv2音声録音は明示拒否する。受入条件とslice順序はcascade設計文書に置く。 -- #18/C2のmetadata-only service観測をlocal candidateとして実装した。v1/v2の観測専用runで、framework非依存の同一DTO/HTTP batchを使用する。collector/event、session/turn/request/parent/response/segment/epochを保持し、本文・URL・raw provider ID・secretのfieldを閉じる。契約は[cascade observation API](docs/cascade-observation-api.md)。 +- #18/C2のmetadata-only service観測をDraft PR #25のreview branch candidateとして実装した。v1/v2の観測専用runで、framework非依存の同一DTO/HTTP batchを使用する。collector/event、session/turn/request/parent/response/segment/epochを保持し、本文・URL・raw provider ID・secretのfieldを閉じる。契約は[cascade observation API](docs/cascade-observation-api.md)。 - 同一event IDの同一payload再送だけを冪等とし、変更は409。未知parentをcoverage不足として残し、out-of-order到着後は再構築する。cancel request/ackとrun completeを別証拠とする。observer queueとrun event数をboundしdrop countを公開する。 -- #19/C3Aの決定論的因果解析をlocal candidateとして実装した。明示的なparent request chainだけを辿り、STT finalization、turn coordination、LLM dispatch/first output/first answer、aggregation、TTS dispatch/first PCM、local playback、end-to-end local writeを分離する。各値はevidence refs、scope、definition version、clock/uncertaintyを持つ。欠測は`unobserved`、未校正clock間と負の順序は`indeterminate`であり、近接timestampによる推測やservice durationの単純加算はしない。契約は[cascade analysis](docs/cascade-analysis.md)。 +- #19/C3Aの決定論的因果解析をDraft PR #25のreview branch candidateとして実装した。明示的なparent request chainだけを辿り、STT finalization、turn coordination、LLM dispatch/first output/first answer、aggregation、TTS dispatch/first PCM、local playback、end-to-end local writeを分離する。各値はevidence refs、scope、definition version、clock/uncertaintyを持つ。欠測は`unobserved`、未校正clock間と負の順序は`indeterminate`であり、近接timestampによる推測やservice durationの単純加算はしない。契約は[cascade analysis](docs/cascade-analysis.md)。 - `llm.first_output`はanswer/reasoning/tool_call/otherの種別だけを保持し、本文を保持しない。tool-call continuationとretryは別operation。generation epoch/responseが異なる古いplaybackをend-to-endに再利用しない。 - latency incidentはv2 configの明示的`service_latency_slos`があるobserved measurementだけに生成する。通常metadataからlatency SLO、STT精度、意味品質、発音品質、remote audible timeを推論しない。 - C3A後もaudio segment mapping、inspector、provider/framework test project・adapter、test-call実行は未実装。v2 `/runs/observed`のみ対応し、`/runs`・`/runs/async`とv2 audio chunkは明示拒否する。 diff --git a/PROGRESS.md b/PROGRESS.md index 1beb009..1288df7 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,6 +2,13 @@ > 本書は日付ごとの履歴。各節の未実装リスト・進捗率は当時のsnapshotであり、現在の状態は [implementation status](docs/implementation-status.md) を参照する。 +## Cascade C1–C3A review publication (2026-09-27) + +- `ae575b6`を基点に、UI/SSE candidateと個人ファイルを除いたC1–C3Aを`agent/cascade-c1-c3`へコミットした。実装commitは`eefae1a`。 +- [Draft PR #25](https://github.com/engkimo/voxbench/pull/25)を`main`向けに作成した。#17、#18、#19を参照し、merge前なのでissueはcloseしていない。 +- PR用checkoutでRuff成功、444 passed / 5 skipped。v1 model/config/manifest schemaは基点とbyte-identical。実Postgres、Pipecat、provider、browserは未検証としてPRに明記した。 +- 次ゴールは同一scenarioと観測契約を使うdirect/no-middleware版とPipecat版の決定論的fake Cascade test project（#22、#23）。 + ## C3A: deterministic Cascade causal latency (2026-09-27) - #19をlocal candidateとして実装。保存済みmetadata-only service eventの明示的turn/request/parent/response/segment/epochだけからoperationを再構築する`cascade_analysis.py`を追加した。 diff --git a/README.md b/README.md index eb18f1e..c41fd7f 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,13 @@ local diagnostic SSE sections below describe an **uncommitted local candidate**, not functionality available in the audited `ae575b6` checkout. Dated progress and memory entries are historical snapshots. -Independent STT → LLM → TTS configuration is implemented as a local candidate -for issue #17: [v2 contract and examples](docs/cascade-config-v2.md). -[Metadata-only service observation](docs/cascade-observation-api.md) is also a -local candidate. [Deterministic causal latency analysis](docs/cascade-analysis.md) -is implemented locally for issue #19. Recording maps, inspection, adapters and -execution remain planned; v2 execution APIs explicitly reject them. +Independent STT → LLM → TTS configuration, metadata-only service observation, +and deterministic causal latency analysis are committed on +[review branch PR #25](https://github.com/engkimo/voxbench/pull/25) for issues +#17–#19. They are not yet merged into `main`; the contracts are documented in the +[v2 guide](docs/cascade-config-v2.md), [observation API](docs/cascade-observation-api.md), +and [analysis guide](docs/cascade-analysis.md). Recording maps, inspection, +adapters and execution remain planned; v2 execution APIs explicitly reject them. Pipecat is optional: the observation library/HTTP ingest and built-in direct realtime bridge do not require it. The cascade target includes Pipecat, other diff --git a/ROADMAP.md b/ROADMAP.md index 94b0ed2..b2a9eb7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -43,9 +43,10 @@ inventory for current availability and verification scope. ## Next: separate STT, LLM and TTS models C1 configuration/capability contracts, C2 metadata-only service observation and -C3A deterministic causal latency analysis -are implemented as local candidates: [current v2 contract](docs/cascade-config-v2.md) -and [service observation API](docs/cascade-observation-api.md), plus the +C3A deterministic causal latency analysis are committed on +[review branch PR #25](https://github.com/engkimo/voxbench/pull/25), pending merge: +[current v2 contract](docs/cascade-config-v2.md), +[service observation API](docs/cascade-observation-api.md), and [causal analysis contract](docs/cascade-analysis.md). Recording maps, inspector support, adapters and execution remain planned. diff --git a/docs/cascade-analysis.md b/docs/cascade-analysis.md index b5b33ce..83246a1 100644 --- a/docs/cascade-analysis.md +++ b/docs/cascade-analysis.md @@ -1,6 +1,7 @@ # Cascade causal latency analysis -Implemented locally for [issue #19](https://github.com/engkimo/voxbench/issues/19). +Implemented on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), +pending review and merge, for [issue #19](https://github.com/engkimo/voxbench/issues/19). The analyzer derives service waits only from explicitly related metadata-only events. It does not inspect conversation text and does not require Pipecat or a provider SDK. diff --git a/docs/cascade-config-v2.md b/docs/cascade-config-v2.md index 2dda580..4d065e6 100644 --- a/docs/cascade-config-v2.md +++ b/docs/cascade-config-v2.md @@ -1,8 +1,9 @@ # Versioned realtime and cascade configuration -Implemented locally for [issue #17](https://github.com/engkimo/voxbench/issues/17). +Implemented on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), +pending review and merge, for [issue #17](https://github.com/engkimo/voxbench/issues/17). This slice validates and resolves configuration. Metadata-only service observation -is now a separate local candidate described in +is part of the same review branch and described in [cascade-observation-api.md](cascade-observation-api.md). V2 recording ingestion, inspector support and test-call execution remain future slices. `/runs` and `/runs/async` reject v2 before creating a run; `/runs/observed` accepts it for diff --git a/docs/cascade-design.md b/docs/cascade-design.md index 104d1a8..cb40d29 100644 --- a/docs/cascade-design.md +++ b/docs/cascade-design.md @@ -1,10 +1,12 @@ # STT → LLM → TTS cascade support: investigation and design -Status: design with C1, C2 and C3A implemented as local candidates, 2026-09-27. +Status: design with C1, C2 and C3A implemented on +[Draft PR #25](https://github.com/engkimo/voxbench/pull/25), pending merge, +2026-09-27. The current C1 shape and examples are in [cascade-config-v2.md](cascade-config-v2.md). The C2 service-event contract is in [cascade-observation-api.md](cascade-observation-api.md). The -[causal analysis contract](cascade-analysis.md) is now implemented locally. Recording +[causal analysis contract](cascade-analysis.md) is implemented on the review branch. Recording maps, adapters and runtime remain planned. Current status is in [implementation-status.md](implementation-status.md). @@ -483,8 +485,9 @@ Each slice must leave v1 realtime/observer workflows working. These cascade slic are distinct from the diagnostic-agent Phase 0–5 roadmap. Published tasks: [issue plan and dependencies](cascade-issue-plan.md), under -[parent issue #15](https://github.com/engkimo/voxbench/issues/15). Publication tracks -planned work; it does not change the implementation status of these proposals. +[parent issue #15](https://github.com/engkimo/voxbench/issues/15). Issue publication +tracks planned work; code status is recorded separately in the implementation +inventory and Draft PR #25. | Slice | Work | Acceptance gate | | --- | --- | --- | @@ -502,10 +505,10 @@ C1–C3 can target the committed observation contracts without depending on the local Ask VoxBench implementation. S0 is required before making that candidate a shipped dependency. Record each implemented slice/commit in the status inventory. -The proposed first cascade implementation PR is C1 only: closed schemas, -registry validation, v1 regression fixtures, examples and updated generated -schemas. C2 then proves metadata-only cascade observation before a live provider -or Web feature is added. +The implementation review branch combines C1–C3A in Draft PR #25: closed +schemas and registry validation, metadata-only observation, and deterministic +causal analysis. It retains the slice boundaries in tests and documentation and +adds no live provider, framework runtime, or Web feature. After C3A, the user-designated next goal is a matched pair of deterministic fake Cascade test applications: one direct application-owned STT/LLM/TTS loop and one diff --git a/docs/cascade-issue-plan.md b/docs/cascade-issue-plan.md index aacd9dd..2fcc537 100644 --- a/docs/cascade-issue-plan.md +++ b/docs/cascade-issue-plan.md @@ -1,8 +1,9 @@ # Cascade and framework-independent issue plan Recorded 2026-09-18. GitHub issues were created using the verified `engkimo` -connector account. Issue publication is complete; feature implementation and -local source/document commit publication are not. +connector account. Issue publication is complete. C1–C3A are committed on +[Draft PR #25](https://github.com/engkimo/voxbench/pull/25), pending review and +merge; this does not make them available on `main`. Parent: [#15: Support framework-independent realtime and STT → LLM → TTS diagnostics](https://github.com/engkimo/voxbench/issues/15) @@ -37,13 +38,13 @@ observation can proceed independently. ## First work and remaining choices -C1/#17, C2/#18 and C3A/#19 are implemented as local candidates. See +C1/#17, C2/#18 and C3A/#19 are review branch candidates in Draft PR #25. See [the v2 config contract](cascade-config-v2.md) and [the service observation contract](cascade-observation-api.md), [the causal analysis contract](cascade-analysis.md), and [the implementation inventory](implementation-status.md) for checks and precise scope. V2 execution and audio recording ingestion remain planned; the GitHub -issues remain open until review/publication. S0 and all other slices retain their +issues remain open until review and merge. S0 and all other slices retain their previous status. Start with #17 for versioned schemas/registry compatibility; develop diff --git a/docs/cascade-observation-api.md b/docs/cascade-observation-api.md index cff6da5..2e640e3 100644 --- a/docs/cascade-observation-api.md +++ b/docs/cascade-observation-api.md @@ -1,6 +1,7 @@ # Metadata-only service observation API -Implemented locally for [issue #18](https://github.com/engkimo/voxbench/issues/18). +Implemented on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), +pending review and merge, for [issue #18](https://github.com/engkimo/voxbench/issues/18). This contract lets direct applications, Pipecat applications and other runtimes report the same observed STT/LLM/TTS/realtime boundaries. It does not require a framework SDK and does not execute providers or retain conversation content. diff --git a/docs/implementation-status.md b/docs/implementation-status.md index fd5c829..cb424e6 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -1,20 +1,23 @@ # VoxBench implementation status Last audited: 2026-09-27. This inventory replaces dated pending lists as the entry -point for current implementation status. It does not turn a local candidate into -a published feature. +point for current implementation status. A review branch candidate is not available +on `main` until its pull request is merged. ## Baseline and status vocabulary -The publication worktree baseline is `ae575b6` (`Detect click/pop discontinuities -with bounded PCM evidence`). -Use `git rev-parse HEAD` and `git status --short` to check whether that baseline -has changed; this document is an audit record, not a live Git query. +The audited `main` baseline is `ae575b6` (`Detect click/pop discontinuities with +bounded PCM evidence`). Cascade C1–C3A are committed as `eefae1a` on +`agent/cascade-c1-c3` and published in +[Draft PR #25](https://github.com/engkimo/voxbench/pull/25). Use +`git rev-parse HEAD` and `git status --short` to identify the checkout; this +document is an audit record, not a live Git query. | Status | Meaning | | --- | --- | | Committed | Implementation exists in the audited Git baseline | -| Local candidate | Implementation exists locally, with verification recorded below, but is absent from the baseline | +| Review branch candidate | Implementation is committed and published for review, but is absent from `main` until merge | +| Local candidate | Implementation exists only in a local working tree, with verification recorded below | | Planned | Design exists; runnable implementation is absent | | Deployment validation pending | A code boundary exists, but the named deployment behavior has not been established by this audit | @@ -25,18 +28,18 @@ current readiness assessment. The framework-independent realtime/cascade plan is tracked by [parent issue #15](https://github.com/engkimo/voxbench/issues/15) and the -[issue dependency plan](cascade-issue-plan.md). C1, C2 and C3A are local candidates; -other tasks remain planned. The -UI/SSE stabilization issue #16 is separate from cascade contract issue #17. +[issue dependency plan](cascade-issue-plan.md). C1, C2 and C3A are review +branch candidates in Draft PR #25; other tasks remain planned. The UI/SSE +stabilization issue #16 is separate from cascade contract issue #17. ## Current inventory | Area | Status | Evidence and remaining boundary | | --- | --- | --- | | Config/manifest schemas, overlays, resolved hashes, static validation | Committed | `schemas.py`, `registry/`; currently one `ai.provider/model`, v1 only | -| V2 realtime/cascade config and versioned capabilities | Local candidate | `schemas_v2.py`, `registry/v2.py`, `registry/config_views.py`, `test_registry_v2.py`; role/format/ID/authority checks, manifest pins, same-mode overlays; [contract guide](cascade-config-v2.md) | -| Framework-independent service event observation | Local candidate | `observability/service_events.py`, observer/HTTP batch extension and `test_service_observations.py`; v1/v2, persistence, causal aliases, privacy, idempotency/conflicts, bounded queues/run events; [HTTP contract](cascade-observation-api.md) | -| Cascade causal grouping and latency analysis | Local candidate | `observability/cascade_analysis.py`, safe run component projections and `test_cascade_analysis.py`; explicit parent chains, endpoint evidence, critical path, clock/missing states and opt-in SLOs; [analysis contract](cascade-analysis.md) | +| V2 realtime/cascade config and versioned capabilities | Review branch candidate | `schemas_v2.py`, `registry/v2.py`, `registry/config_views.py`, `test_registry_v2.py`; role/format/ID/authority checks, manifest pins, same-mode overlays; [contract guide](cascade-config-v2.md) | +| Framework-independent service event observation | Review branch candidate | `observability/service_events.py`, observer/HTTP batch extension and `test_service_observations.py`; v1/v2, persistence, causal aliases, privacy, idempotency/conflicts, bounded queues/run events; [HTTP contract](cascade-observation-api.md) | +| Cascade causal grouping and latency analysis | Review branch candidate | `observability/cascade_analysis.py`, safe run component projections and `test_cascade_analysis.py`; explicit parent chains, endpoint evidence, critical path, clock/missing states and opt-in SLOs; [analysis contract](cascade-analysis.md) | | Harness WAV taps and OTel spans | Committed | `engine_harness/`; the default `run_once` generates nominal artifacts, not a real provider conversation | | Signal invariants and synthetic full-reference scoring | Committed | `verification/`, `synthetic_caller/`; optional ViSQOL CLI, explicit blocked/unavailable states, aggregation/regression/calibration | | Common timeline and deterministic incidents | Committed | `run_api.py`, `tests/test_timeline_stage_diagnostics.py`; five primitives, clock uncertainty, explicit observation boundaries | @@ -54,7 +57,7 @@ UI/SSE stabilization issue #16 is separate from cascade contract issue #17. | Deterministic diagnostic sessions, finite SSE replay and result acknowledgement | Local candidate | Exact files below; process-local store, one selected incident/event, no external model | | Session/event/result persistence, model orchestration, diagnostic jobs/cancel, guided sequence | Planned | `diagnostic-agent-design.md`; not completed by the local SSE slice | | Product-wide OIDC authentication, authorization, audit/retention | Planned | `diagnostic-agent-design.md`; required before production diagnostic integration | -| Cascade recording maps, inspector, test projects, adapters and runtime | Planned | `cascade-design.md`; causal analysis is local, while v2 audio chunks and execution remain explicitly unsupported. Next goal: matched direct/Pipecat fake Cascade projects for tuning and evaluation | +| Cascade recording maps, inspector, test projects, adapters and runtime | Planned | `cascade-design.md`; causal analysis is in Draft PR #25, while v2 audio chunks and execution remain explicitly unsupported. Next goal: matched direct/Pipecat fake Cascade projects for tuning and evaluation | | Raw PCAP import, log/source snapshot connectors, second engine, scale profile | Planned | Designs/roadmap; not established by the RTP tap or second realtime provider | ## Local UI/SSE candidate audit @@ -145,16 +148,17 @@ Historical real-call evidence is not a substitute for those checks. own acceptance criteria before production use. 5. Commit the reviewed change as its own unit when implementation stabilization is requested, then update this inventory with its commit and verification. - C1 implementation has not committed or published the UI/SSE candidate. + Draft PR #25 does not include or publish the UI/SSE candidate. Cascade contracts are implemented against the committed v1 contracts independently of that step. The local UI dispatcher is an optional integration candidate with a separate stabilization dependency, rather than an implicit released dependency. -## C1 config/capability local candidate (2026-09-18) +## C1 config/capability review branch candidate (2026-09-18) -Scope: [#17](https://github.com/engkimo/voxbench/issues/17), implemented locally, -not committed or published. The issue remains open for review/publication. +Scope: [#17](https://github.com/engkimo/voxbench/issues/17), committed in `eefae1a` +on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), pending merge. +The issue remains open. [cascade-config-v2.md](cascade-config-v2.md) records the implemented contract. Required new files: @@ -198,10 +202,11 @@ output. C3A now derives causal timing; recording mapping, UI, framework/provider test projects/adapters and execution remain separate future slices. Validation does not establish live provider/model readiness or complete observation coverage. -## C2 service-observation local candidate (2026-09-27) +## C2 service-observation review branch candidate (2026-09-27) -Scope: [#18](https://github.com/engkimo/voxbench/issues/18), implemented locally, -not committed or published. The issue remains open for review/publication. +Scope: [#18](https://github.com/engkimo/voxbench/issues/18), committed in `eefae1a` +on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), pending merge. +The issue remains open. [cascade-observation-api.md](cascade-observation-api.md) is the normative HTTP, privacy, identity and retry guide. @@ -235,10 +240,11 @@ actual Pipecat/other-framework integration was performed. C2 normalizes observed metadata; C3A now derives service latency, while audio segment mapping, remote playout and provider readiness remain unestablished. -## C3A causal-analysis local candidate (2026-09-27) +## C3A causal-analysis review branch candidate (2026-09-27) -Scope: [#19](https://github.com/engkimo/voxbench/issues/19), implemented locally, -not committed or published. The issue remains open for review/publication. +Scope: [#19](https://github.com/engkimo/voxbench/issues/19), committed in `eefae1a` +on [Draft PR #25](https://github.com/engkimo/voxbench/pull/25), pending merge. +The issue remains open. [cascade-analysis.md](cascade-analysis.md) records the measurement, clock, critical-path, coverage and SLO contract.