From 5efead626dfad4cfa028c73089c806847230e995 Mon Sep 17 00:00:00 2001 From: Jake Lasky Date: Wed, 18 Mar 2026 15:45:10 -0400 Subject: [PATCH 01/10] feat(tts): add ElevenLabs TTS provider and Aura-2 voice dropdown - Add ElevenLabs TTS backend (API-only, hidden from UI) - /api/tts-voices endpoint to list ElevenLabs voices - /api/tts-transcribe accepts tts_provider param - Replace free-text voice input with language-filtered dropdown - 7 language tabs: English, Spanish, German, French, Italian, Dutch, Japanese - 90+ Deepgram Aura-2 voices with name, accent, and gender labels - Defaults to English / Asteria on load Co-Authored-By: Claude Opus 4.6 --- app.py | 80 ++++++++++++++++++++- static/app.js | 166 +++++++++++++++++++++++++++++++++++++++++++ templates/index.html | 26 ++++++- 3 files changed, 266 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index ed06bce..31e2a35 100644 --- a/app.py +++ b/app.py @@ -94,6 +94,25 @@ async def _tts_generate(text: str, tts_model: str, api_key: str) -> bytes: return resp.content +async def _elevenlabs_tts_generate(text: str, voice_id: str, api_key: str) -> bytes: + """Call ElevenLabs TTS and return MP3 bytes.""" + async with httpx.AsyncClient(timeout=60.0) as client: + resp = await client.post( + f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}", + headers={ + "xi-api-key": api_key, + "Content-Type": "application/json", + "Accept": "audio/mpeg", + }, + json={ + "text": text, + "model_id": "eleven_multilingual_v2", + }, + ) + resp.raise_for_status() + return resp.content + + async def _stt_batch(audio_bytes: bytes, stt_params: dict, api_key: str) -> dict: """Transcribe audio bytes via Deepgram pre-recorded (batch) API.""" headers = {"Authorization": f"Token {api_key}"} @@ -163,11 +182,54 @@ async def on_message(msg, **kwargs): } +@fastapi_app.get("/api/tts-voices") +async def tts_voices(provider: str = "elevenlabs"): + """Return available voices for a TTS provider.""" + if provider == "elevenlabs": + api_key = os.getenv("ELEVENLABS_API_KEY", "") + if not api_key: + return JSONResponse({"error": "ELEVENLABS_API_KEY not set"}, status_code=500) + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.get( + "https://api.elevenlabs.io/v1/voices", + headers={"xi-api-key": api_key}, + ) + resp.raise_for_status() + data = resp.json() + voices = [ + { + "voice_id": v["voice_id"], + "name": v["name"], + "accent": v.get("labels", {}).get("accent", ""), + "gender": v.get("labels", {}).get("gender", ""), + "age": v.get("labels", {}).get("age", ""), + "description": v.get("labels", {}).get("description", ""), + "use_case": v.get("labels", {}).get("use_case", ""), + } + for v in data.get("voices", []) + ] + return JSONResponse({"voices": voices}) + return JSONResponse({"error": f"Unknown provider: {provider}"}, status_code=400) + + +async def _generate_tts_audio(text: str, tts_model: str, provider: str) -> bytes: + """Route TTS generation to the correct provider.""" + if provider == "elevenlabs": + api_key = os.getenv("ELEVENLABS_API_KEY", "") + if not api_key: + raise ValueError("ELEVENLABS_API_KEY not set") + return await _elevenlabs_tts_generate(text, tts_model, api_key) + else: + api_key = os.getenv("DEEPGRAM_API_KEY", "") + return await _tts_generate(text, tts_model, api_key) + + @fastapi_app.post("/api/tts-transcribe") async def tts_transcribe(request: Request): body = await request.json() text = body.get("text", "").strip() tts_model = body.get("tts_model", "aura-2-asteria-en") + tts_provider = body.get("tts_provider", "deepgram") stt_params = body.get("stt_params", {}) mode = body.get("mode", "batch") # "batch" | "streaming" | "both" @@ -180,17 +242,29 @@ async def tts_transcribe(request: Request): try: if mode == "batch": - audio_bytes = await _tts_generate(text, tts_model, api_key) + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) result = await _stt_batch(audio_bytes, stt_params, api_key) return JSONResponse(result) elif mode == "streaming": + if tts_provider == "elevenlabs": + # ElevenLabs doesn't integrate with Deepgram streaming pipe, + # so generate full audio first, then stream it to STT + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + result = await _stt_batch(audio_bytes, stt_params, api_key) + return JSONResponse(result) result = await _stt_streaming(text, tts_model, stt_params, api_key) return JSONResponse(result) - else: # both — batch buffers TTS bytes, streaming pipes TTS live; run in parallel + else: # both + if tts_provider == "elevenlabs": + # Can't do streaming pipe with ElevenLabs, run batch only + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + result = await _stt_batch(audio_bytes, stt_params, api_key) + return JSONResponse(result) + async def _batch_pipeline(): - audio_bytes = await _tts_generate(text, tts_model, api_key) + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) return await _stt_batch(audio_bytes, stt_params, api_key) batch_result, stream_result = await asyncio.gather( diff --git a/static/app.js b/static/app.js index 1fbea16..9dbe2ec 100644 --- a/static/app.js +++ b/static/app.js @@ -26,13 +26,132 @@ function appData() { // TTS Test ttsText: '', + ttsProvider: 'deepgram', // 'deepgram' | 'elevenlabs' ttsModel: 'aura-2-asteria-en', + ttsLang: 'en', ttsMode: 'batch', // 'batch' | 'streaming' | 'both' ttsLoading: false, ttsResult: null, ttsLastText: '', ttsLastTranscript: '', ttsLastStreamTranscript: '', + // ElevenLabs voices cache + elevenVoices: [], + elevenVoicesLoading: false, + + // Deepgram Aura-2 voices grouped by language + dgVoiceLangs: [ + { code: 'en', label: 'English' }, + { code: 'es', label: 'Spanish' }, + { code: 'de', label: 'German' }, + { code: 'fr', label: 'French' }, + { code: 'it', label: 'Italian' }, + { code: 'nl', label: 'Dutch' }, + { code: 'ja', label: 'Japanese' }, + ], + dgVoices: [ + // English — American + { id: 'aura-2-asteria-en', name: 'Asteria', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-andromeda-en', name: 'Andromeda', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-apollo-en', name: 'Apollo', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-arcas-en', name: 'Arcas', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-aries-en', name: 'Aries', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-athena-en', name: 'Athena', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-atlas-en', name: 'Atlas', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-aurora-en', name: 'Aurora', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-callista-en', name: 'Callista', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-cora-en', name: 'Cora', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-cordelia-en', name: 'Cordelia', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-delia-en', name: 'Delia', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-electra-en', name: 'Electra', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-harmonia-en', name: 'Harmonia', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-helena-en', name: 'Helena', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-hera-en', name: 'Hera', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-hermes-en', name: 'Hermes', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-iris-en', name: 'Iris', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-juno-en', name: 'Juno', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-jupiter-en', name: 'Jupiter', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-luna-en', name: 'Luna', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-mars-en', name: 'Mars', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-minerva-en', name: 'Minerva', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-neptune-en', name: 'Neptune', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-odysseus-en', name: 'Odysseus', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-ophelia-en', name: 'Ophelia', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-orion-en', name: 'Orion', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-orpheus-en', name: 'Orpheus', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-phoebe-en', name: 'Phoebe', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-pluto-en', name: 'Pluto', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-saturn-en', name: 'Saturn', gender: 'M', accent: 'American', lang: 'en' }, + { id: 'aura-2-selene-en', name: 'Selene', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-thalia-en', name: 'Thalia', gender: 'F', accent: 'American', lang: 'en' }, + { id: 'aura-2-zeus-en', name: 'Zeus', gender: 'M', accent: 'American', lang: 'en' }, + // English — Southern US + { id: 'aura-2-janus-en', name: 'Janus', gender: 'F', accent: 'Southern US', lang: 'en' }, + // English — Filipino + { id: 'aura-2-amalthea-en', name: 'Amalthea', gender: 'F', accent: 'Filipino', lang: 'en' }, + // English — British + { id: 'aura-2-draco-en', name: 'Draco', gender: 'M', accent: 'British', lang: 'en' }, + { id: 'aura-2-pandora-en', name: 'Pandora', gender: 'F', accent: 'British', lang: 'en' }, + // English — Australian + { id: 'aura-2-hyperion-en', name: 'Hyperion', gender: 'M', accent: 'Australian', lang: 'en' }, + { id: 'aura-2-theia-en', name: 'Theia', gender: 'F', accent: 'Australian', lang: 'en' }, + // Spanish + { id: 'aura-2-estrella-es', name: 'Estrella', gender: 'F', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-sirio-es', name: 'Sirio', gender: 'M', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-javier-es', name: 'Javier', gender: 'M', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-luciano-es', name: 'Luciano', gender: 'M', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-olivia-es', name: 'Olivia', gender: 'F', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-valerio-es', name: 'Valerio', gender: 'M', accent: 'Mexican', lang: 'es' }, + { id: 'aura-2-nestor-es', name: 'Nestor', gender: 'M', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-carina-es', name: 'Carina', gender: 'F', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-alvaro-es', name: 'Alvaro', gender: 'M', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-diana-es', name: 'Diana', gender: 'F', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-agustina-es', name: 'Agustina', gender: 'F', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-silvia-es', name: 'Silvia', gender: 'F', accent: 'Peninsular', lang: 'es' }, + { id: 'aura-2-celeste-es', name: 'Celeste', gender: 'F', accent: 'Colombian', lang: 'es' }, + { id: 'aura-2-gloria-es', name: 'Gloria', gender: 'F', accent: 'Colombian', lang: 'es' }, + { id: 'aura-2-antonia-es', name: 'Antonia', gender: 'F', accent: 'Argentine', lang: 'es' }, + { id: 'aura-2-aquila-es', name: 'Aquila', gender: 'M', accent: 'Latin American', lang: 'es' }, + { id: 'aura-2-selena-es', name: 'Selena', gender: 'F', accent: 'Latin American', lang: 'es' }, + // German + { id: 'aura-2-julius-de', name: 'Julius', gender: 'M', accent: 'German', lang: 'de' }, + { id: 'aura-2-viktoria-de', name: 'Viktoria', gender: 'F', accent: 'German', lang: 'de' }, + { id: 'aura-2-elara-de', name: 'Elara', gender: 'F', accent: 'German', lang: 'de' }, + { id: 'aura-2-aurelia-de', name: 'Aurelia', gender: 'F', accent: 'German', lang: 'de' }, + { id: 'aura-2-lara-de', name: 'Lara', gender: 'F', accent: 'German', lang: 'de' }, + { id: 'aura-2-fabian-de', name: 'Fabian', gender: 'M', accent: 'German', lang: 'de' }, + { id: 'aura-2-kara-de', name: 'Kara', gender: 'F', accent: 'German', lang: 'de' }, + // French + { id: 'aura-2-agathe-fr', name: 'Agathe', gender: 'F', accent: 'French', lang: 'fr' }, + { id: 'aura-2-hector-fr', name: 'Hector', gender: 'M', accent: 'French', lang: 'fr' }, + // Italian + { id: 'aura-2-livia-it', name: 'Livia', gender: 'F', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-dionisio-it', name: 'Dionisio', gender: 'M', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-melia-it', name: 'Melia', gender: 'F', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-elio-it', name: 'Elio', gender: 'M', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-flavio-it', name: 'Flavio', gender: 'M', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-maia-it', name: 'Maia', gender: 'F', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-cinzia-it', name: 'Cinzia', gender: 'F', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-cesare-it', name: 'Cesare', gender: 'M', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-perseo-it', name: 'Perseo', gender: 'M', accent: 'Italian', lang: 'it' }, + { id: 'aura-2-demetra-it', name: 'Demetra', gender: 'F', accent: 'Italian', lang: 'it' }, + // Dutch + { id: 'aura-2-rhea-nl', name: 'Rhea', gender: 'F', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-sander-nl', name: 'Sander', gender: 'M', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-beatrix-nl', name: 'Beatrix', gender: 'F', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-daphne-nl', name: 'Daphne', gender: 'F', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-cornelia-nl', name: 'Cornelia', gender: 'F', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-hestia-nl', name: 'Hestia', gender: 'F', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-lars-nl', name: 'Lars', gender: 'M', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-roman-nl', name: 'Roman', gender: 'M', accent: 'Dutch', lang: 'nl' }, + { id: 'aura-2-leda-nl', name: 'Leda', gender: 'F', accent: 'Dutch', lang: 'nl' }, + // Japanese + { id: 'aura-2-fujin-ja', name: 'Fujin', gender: 'M', accent: 'Japanese', lang: 'ja' }, + { id: 'aura-2-izanami-ja', name: 'Izanami', gender: 'F', accent: 'Japanese', lang: 'ja' }, + { id: 'aura-2-uzume-ja', name: 'Uzume', gender: 'F', accent: 'Japanese', lang: 'ja' }, + { id: 'aura-2-ebisu-ja', name: 'Ebisu', gender: 'M', accent: 'Japanese', lang: 'ja' }, + { id: 'aura-2-ama-ja', name: 'Ama', gender: 'F', accent: 'Japanese', lang: 'ja' }, + ], // Transcript finalTranscript: '', @@ -432,6 +551,47 @@ function appData() { } }, + filteredDgVoices() { + return this.dgVoices.filter(v => v.lang === this.ttsLang); + }, + + switchTtsLang(lang) { + this.ttsLang = lang; + const voices = this.dgVoices.filter(v => v.lang === lang); + if (voices.length && !voices.find(v => v.id === this.ttsModel)) { + this.ttsModel = voices[0].id; + } + }, + + async loadElevenVoices() { + if (this.elevenVoices.length > 0) return; + this.elevenVoicesLoading = true; + try { + const res = await fetch('/api/tts-voices?provider=elevenlabs'); + const data = await res.json(); + if (data.error) throw new Error(data.error); + this.elevenVoices = data.voices || []; + } catch (err) { + this.showToast('Failed to load ElevenLabs voices: ' + err.message, 'error'); + } finally { + this.elevenVoicesLoading = false; + } + }, + + switchTtsProvider(provider) { + this.ttsProvider = provider; + if (provider === 'elevenlabs') { + this.loadElevenVoices(); + if (!this.ttsModel || this.ttsModel.startsWith('aura-')) { + this.ttsModel = ''; + } + } else { + if (!this.ttsModel || !this.ttsModel.startsWith('aura-')) { + this.ttsModel = 'aura-2-asteria-en'; + } + } + }, + async runTts() { if (!this.ttsText.trim()) return; this.ttsLoading = true; @@ -445,6 +605,7 @@ function appData() { body: JSON.stringify({ text: this.ttsText, tts_model: this.ttsModel, + tts_provider: this.ttsProvider, mode: this.ttsMode, stt_params: this.getCleanParams('batch'), }), @@ -552,6 +713,11 @@ function appData() { const base = this.params.base_url || 'api.deepgram.com'; if (this.mode === 'tts') { + if (this.ttsProvider === 'elevenlabs') { + const voiceId = this.ttsModel || '(select voice)'; + this.urlDisplay = `api.elevenlabs.io/v1/text-to-speech/${voiceId}`; + return `https://${this.urlDisplay}`; + } const ttsModel = this.ttsModel || 'aura-2-asteria-en'; this.urlDisplay = `${base}/v1/speak?model=${ttsModel}&encoding=mp3`; return `https://${this.urlDisplay}`; diff --git a/templates/index.html b/templates/index.html index d608da2..d099b57 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1097,8 +1097,28 @@ style="resize:vertical;min-height:80px;font-family:inherit;">
-
TTS Voice
- +
Language
+
+ +
+
+
+
Voice
+
STT Mode
@@ -1114,7 +1134,7 @@
+ + From bcf4562be6d3a12c020cfd6f5e8bdf0a3a569f91 Mon Sep 17 00:00:00 2001 From: Jake Lasky Date: Sat, 25 Jul 2026 00:59:17 -0400 Subject: [PATCH 03/10] fix(deps): drop unused av, demote websocket-client to a dev dependency av was added but nothing imports it. websocket-client is only reachable from stt/client.py's sync STTClient, which only scripts/test_redaction.py uses, so it is dev tooling rather than an app runtime dependency. Promoting it to [project] dependencies broke the v2 migration guard, which asserted the string was absent from the whole pyproject. That assertion was too broad: the real constraint is that the served async app never ships the sync gevent-era WebSocket path. Retarget the test at [project] dependencies via tomllib and rename it accordingly. Also replace the hardcoded /coding/... absolute paths in this file with a REPO_ROOT derived from __file__, which pinned the suite to one machine. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 4 ++++ tests/test_streaming.py | 32 ++++++++++++++++++++++++-------- uv.lock | 11 +++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d420ef2..5bf1197 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,10 @@ dev-dependencies = [ "aiohttp>=3.9.0", "pytest>=8.0", "pytest-asyncio>=0.23,<1.0", + # scripts/ only, for stt/client.py's sync STTClient. DO NOT promote to + # [project] dependencies: the async app must never ship the sync + # websocket-client path again (see test_websocket_client_not_a_runtime_dep). + "websocket-client>=1.9.0", ] [tool.pytest.ini_options] diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 29ed401..280f2ae 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -87,26 +87,42 @@ def __init__(self, mock_ws=None): # Static / structural tests (no server needed) # --------------------------------------------------------------------------- +# Repo root, derived from this file's location. DO NOT hardcode an absolute +# path here: it pins the suite to one machine and fails on every other clone. +REPO_ROOT = Path(__file__).resolve().parents[1] + + def test_no_websocket_client_import(): - """app.py must NOT import websocket-client; must use deepgram SDK instead.""" - app_text = Path("/coding/deepgram-python-stt/app.py").read_text() - assert "import websocket" not in app_text, "websocket-client import found in app.py" + """app.py must NOT import websocket-client (sync); websockets (async) is OK for custom endpoints.""" + app_text = (REPO_ROOT / "app.py").read_text() + assert "import websocket\n" not in app_text and "from websocket " not in app_text, \ + "websocket-client (sync) import found in app.py" assert "AsyncDeepgramClient" in app_text, "AsyncDeepgramClient not found in app.py" def test_no_threading_in_app(): """app.py must not import threading or use time.sleep in streaming path.""" - app_text = Path("/coding/deepgram-python-stt/app.py").read_text() + app_text = (REPO_ROOT / "app.py").read_text() assert "import threading" not in app_text, "import threading found in app.py" assert "threading.Thread" not in app_text, "threading.Thread found in app.py" assert "threading.Event" not in app_text, "threading.Event found in app.py" assert "time.sleep" not in app_text, "time.sleep found in app.py" -def test_websocket_client_not_in_pyproject(): - """websocket-client must not be a dependency.""" - pyproject = Path("/coding/deepgram-python-stt/pyproject.toml").read_text() - assert "websocket-client" not in pyproject, "websocket-client found in pyproject.toml" +def test_websocket_client_not_a_runtime_dep(): + """websocket-client must not be a RUNTIME dependency of the async app. + + It is legitimately a dev dependency: stt/client.py's sync STTClient still + uses it and scripts/test_redaction.py still imports that. The v2 constraint + is that the served app never ships the sync gevent-era WebSocket path, so + assert on [project] dependencies only, not on the whole file. + """ + import tomllib + + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode()) + runtime_deps = pyproject["project"]["dependencies"] + offenders = [d for d in runtime_deps if "websocket-client" in d] + assert not offenders, f"websocket-client is a runtime dependency: {offenders}" def test_sessions_dict_exists(): diff --git a/uv.lock b/uv.lock index f0b494d..48e5eca 100644 --- a/uv.lock +++ b/uv.lock @@ -215,6 +215,7 @@ dev = [ { name = "aiohttp" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "websocket-client" }, ] [package.metadata] @@ -236,6 +237,7 @@ dev = [ { name = "aiohttp", specifier = ">=3.9.0" }, { name = "pytest", specifier = ">=8.0" }, { name = "pytest-asyncio", specifier = ">=0.23,<1.0" }, + { name = "websocket-client", specifier = ">=1.9.0" }, ] [[package]] @@ -621,6 +623,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/e4/d04a086285c20886c0daad0e026f250869201013d18f81d9ff5eada73a88/uvicorn-0.41.0-py3-none-any.whl", hash = "sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187", size = 68783, upload-time = "2026-02-16T23:07:22.357Z" }, ] +[[package]] +name = "websocket-client" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/41/aa4bf9664e4cda14c3b39865b12251e8e7d239f4cd0e3cc1b6c2ccde25c1/websocket_client-1.9.0.tar.gz", hash = "sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98", size = 70576, upload-time = "2025-10-07T21:16:36.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, +] + [[package]] name = "websockets" version = "15.0.1" From babbdc356ae0e95f5ce2af558ad13563f05456c0 Mon Sep 17 00:00:00 2001 From: Jake Lasky Date: Sat, 25 Jul 2026 01:05:01 -0400 Subject: [PATCH 04/10] fix(security): stop base_url leaking the server API key, contain uploads Three issues on a public unauthenticated app, all verified exploitable against a local server on 2026-07-25 and all live in production. 1. base_url forwarded DEEPGRAM_API_KEY to any host a caller named. base_url was interpolated raw into f"https://{base_url}/v1/listen" alongside an Authorization header, so it controlled the destination host AND could inject path and query. A request with base_url="postman-echo.com/post?x=" delivered "Token " to that third party. Replaced with _resolve_stt_host(), which requires a bare hostname with optional port and exact membership in an operator-controlled ALLOWED_STT_HOSTS env allowlist. The aiworks workflow is preserved by adding that host to the env var; what is gone is caller-chosen destinations. 2. /upload wrote anywhere the process could write. path = TEMP_DIR / file.filename let an absolute filename replace the base entirely (Path("/tmp/x") / "/etc/passwd" is "/etc/passwd") and let ../ traverse out. In the container that is arbitrary overwrite of app code. Now routed through _safe_temp_path(), which takes the basename only and asserts the resolved parent is TEMP_DIR. /files reads go through the same guard. 3. callback was forwarded to Deepgram, which POSTs the finished transcript to a URL of the caller's choosing on this account: data exfil plus arbitrary async jobs. Added DENIED_PARAMS, applied both in the filter loop and again to the extra dict, because that merge runs afterward and would otherwise smuggle it straight through. Also adds spend and disk caps (MAX_TTS_CHARS, MAX_UPLOAD_BYTES) since nothing authenticates callers, and routes /transcribe's error paths through _clean_error so they scrub credentials like the TTS route does. Does NOT close the remaining hole: every endpoint is still unauthenticated. Caps and allowlists blunt the abuse, they do not stop someone spending the key's budget. Co-Authored-By: Claude Opus 5 (1M context) --- app.py | 1569 +++++++++++++++++++++------------------- stt/options.py | 20 +- tests/test_security.py | 130 ++++ 3 files changed, 973 insertions(+), 746 deletions(-) create mode 100644 tests/test_security.py diff --git a/app.py b/app.py index 954af4e..eebd836 100644 --- a/app.py +++ b/app.py @@ -1,743 +1,826 @@ -import asyncio -import json as json_mod -import logging -import os -import re -import tempfile -import urllib.parse -from pathlib import Path - -from mutagen import File as MutagenFile - -import httpx -import socketio -import websockets -from deepgram import AsyncDeepgramClient -from deepgram.core.events import EventType -from deepgram.listen.v1.types import ListenV1Results, ListenV1Metadata -from dotenv import load_dotenv -from fastapi import FastAPI, Request, UploadFile, File -from fastapi.responses import FileResponse, JSONResponse -from fastapi.staticfiles import StaticFiles - -from stt.options import clean_params, Mode - -load_dotenv() - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -PORT = int(os.getenv("PORT", 8001)) -TEMP_DIR = Path(tempfile.gettempdir()) / "deepgram-stt" -TEMP_DIR.mkdir(exist_ok=True) - -# 1. AsyncServer — async_mode MUST be "asgi" (not "gevent", not "threading") -sio = socketio.AsyncServer( - async_mode="asgi", - cors_allowed_origins="*", -) - -# 2. FastAPI sub-app for HTTP routes only -fastapi_app = FastAPI() -fastapi_app.mount("/static", StaticFiles(directory="static"), name="static") - -# 3. Combined ASGI callable — THIS is what uvicorn serves, not fastapi_app -app = socketio.ASGIApp(sio, fastapi_app) - -def _clean_error(e: Exception) -> str: - """Strip SDK request headers (including auth token) from Deepgram exception messages.""" - msg = str(e) - m = re.search(r'status_code:\s*(\d+),\s*body:\s*(.+)$', msg, re.DOTALL) - if m: - return f"Deepgram {m.group(1)}: {m.group(2).strip()}" - return msg - - -# Per-session state — module-level dict, not sio.session() (too slow for audio hot path) -# Key: SocketIO session id (sid) -# Value: dict with keys: task (asyncio.Task), stop_event (asyncio.Event), -# ws (AsyncV1SocketClient | None), request_id (str | None) -_sessions: dict[str, dict] = {} - - -# --- HTTP Routes --- - -@fastapi_app.get("/") -async def index(): - return FileResponse("templates/index.html") - - -@fastapi_app.post("/upload") -async def upload(file: UploadFile = File(...)): - path = TEMP_DIR / file.filename - content = await file.read() - path.write_bytes(content) - return JSONResponse({"filename": file.filename, "size": path.stat().st_size}) - - -@fastapi_app.get("/files/{filename}") -async def serve_file(filename: str): - path = TEMP_DIR / filename - if not path.exists(): - return JSONResponse({"error": "not found"}, status_code=404) - return FileResponse(path) - - -async def _tts_generate(text: str, tts_model: str, api_key: str) -> bytes: - """Call Deepgram TTS and return MP3 bytes.""" - headers = {"Authorization": f"Token {api_key}"} - async with httpx.AsyncClient(timeout=60.0) as client: - resp = await client.post( - "https://api.deepgram.com/v1/speak", - headers={**headers, "Content-Type": "application/json"}, - params={"model": tts_model, "encoding": "mp3"}, - json={"text": text}, - ) - resp.raise_for_status() - return resp.content - - -async def _elevenlabs_tts_generate(text: str, voice_id: str, api_key: str) -> bytes: - """Call ElevenLabs TTS and return MP3 bytes.""" - async with httpx.AsyncClient(timeout=60.0) as client: - resp = await client.post( - f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}", - headers={ - "xi-api-key": api_key, - "Content-Type": "application/json", - "Accept": "audio/mpeg", - }, - json={ - "text": text, - "model_id": "eleven_multilingual_v2", - }, - ) - resp.raise_for_status() - return resp.content - - -async def _stt_batch(audio_bytes: bytes, stt_params: dict, api_key: str) -> dict: - """Transcribe audio bytes via Deepgram pre-recorded (batch) API.""" - headers = {"Authorization": f"Token {api_key}"} - base_url = stt_params.get("base_url", "api.deepgram.com") - clean = clean_params(stt_params, Mode.BATCH) - query_params = {} - for k, v in clean.items(): - if isinstance(v, bool): - query_params[k] = "true" if v else "false" - elif isinstance(v, (list, str)): - query_params[k] = v - else: - query_params[k] = str(v) - query_params.setdefault("model", "nova-2") - - async with httpx.AsyncClient(timeout=60.0) as client: - resp = await client.post( - f"https://{base_url}/v1/listen", - headers={**headers, "Content-Type": "audio/mp3"}, - params=query_params, - content=audio_bytes, - ) - resp.raise_for_status() - return resp.json() - - -async def _stt_streaming(text: str, tts_model: str, stt_params: dict, api_key: str) -> dict: - """Pipe Deepgram TTS streaming response directly into the STT WebSocket. - TTS audio chunks are forwarded to STT as they arrive — naturally paced at - speech speed, no buffering or artificial timing needed. - Returns {"transcript": str, "segments": list, "raw_responses": list}. - """ - base_url = stt_params.get("base_url", "api.deepgram.com") - headers = {"Authorization": f"Token {api_key}"} - - # For custom endpoints (e.g. aiworks), use raw websockets instead of the SDK - if base_url != "api.deepgram.com": - return await _stt_streaming_raw(text, tts_model, stt_params, api_key) - - dg = AsyncDeepgramClient(api_key=api_key) - sdk_kwargs = _params_to_sdk_kwargs(stt_params) - - segments = [] - - async with dg.listen.v1.connect(**sdk_kwargs) as ws: - async def on_message(msg, **kwargs): - if isinstance(msg, ListenV1Results) and bool(msg.is_final): - t = msg.channel.alternatives[0].transcript - if t.strip(): - segments.append(t) - - ws.on(EventType.MESSAGE, on_message) - listen_task = asyncio.create_task(ws.start_listening()) - - # Stream TTS audio directly into STT WebSocket as chunks arrive - async with httpx.AsyncClient(timeout=60.0) as client: - async with client.stream( - "POST", - "https://api.deepgram.com/v1/speak", - headers={**headers, "Content-Type": "application/json"}, - params={"model": tts_model, "encoding": "mp3"}, - json={"text": text}, - ) as tts_resp: - tts_resp.raise_for_status() - async for chunk in tts_resp.aiter_bytes(chunk_size=4096): - await ws.send_media(chunk) - - await ws.send_close_stream() - await listen_task - - return { - "transcript": " ".join(segments), - "segments": segments, - } - - -async def _stt_streaming_raw(text: str, tts_model: str, stt_params: dict, api_key: str) -> dict: - """Stream TTS audio into a custom WebSocket STT endpoint (e.g. aiworks). - Uses raw websockets since the Deepgram SDK only connects to api.deepgram.com. - Returns full raw responses so callers can inspect the response schema. - """ - base_url = stt_params.get("base_url", "api.deepgram.com") - clean = clean_params(stt_params, Mode.STREAMING) - query_parts = [] - for k, v in clean.items(): - if isinstance(v, bool): - query_parts.append(f"{k}={'true' if v else 'false'}") - elif isinstance(v, list): - for item in v: - query_parts.append(f"{k}={urllib.parse.quote(str(item))}") - else: - query_parts.append(f"{k}={urllib.parse.quote(str(v))}") - qs = "&".join(query_parts) - ws_url = f"wss://{base_url}/v1/listen?{qs}" if qs else f"wss://{base_url}/v1/listen" - headers = {"Authorization": f"Token {api_key}"} - - # Generate TTS audio first (full buffer), then stream into WebSocket - async with httpx.AsyncClient(timeout=60.0) as client: - tts_resp = await client.post( - "https://api.deepgram.com/v1/speak", - headers={**headers, "Content-Type": "application/json"}, - params={"model": tts_model, "encoding": "mp3"}, - json={"text": text}, - ) - tts_resp.raise_for_status() - audio_bytes = tts_resp.content - - segments = [] - raw_responses = [] - - async with websockets.connect(ws_url, additional_headers=headers) as ws: - # Send audio in chunks - chunk_size = 4096 - for i in range(0, len(audio_bytes), chunk_size): - await ws.send(audio_bytes[i:i + chunk_size]) - await asyncio.sleep(0.01) - - # Signal end of audio - await ws.send(json_mod.dumps({"type": "CloseStream"})) - - # Collect all responses until connection closes - try: - async for msg in ws: - if isinstance(msg, str): - data = json_mod.loads(msg) - raw_responses.append(data) - # Extract transcript from various response shapes - if "deepgram_stt" in data: - stt = data["deepgram_stt"] - if isinstance(stt, list): - stt = stt[0] - t = stt.get("channel", {}).get("alternatives", [{}])[0].get("transcript", "") - is_final = stt.get("is_final", False) - if t.strip() and is_final: - segments.append(t) - elif data.get("type") == "Results": - t = data.get("channel", {}).get("alternatives", [{}])[0].get("transcript", "") - if t.strip() and data.get("is_final"): - segments.append(t) - elif data.get("type") == "text": - t = data.get("data", "") - if t.strip(): - segments.append(t) - # Top-level transcript (aiworks wrapper) - elif "transcript" in data and not any(k in data for k in ("deepgram_stt", "type")): - t = data["transcript"] - if t.strip(): - segments.append(t) - except websockets.exceptions.ConnectionClosed: - pass - - return { - "transcript": " ".join(segments), - "segments": segments, - "raw_responses": raw_responses, - } - - -@fastapi_app.get("/api/tts-voices") -async def tts_voices(provider: str = "elevenlabs", language: str = ""): - """Return available voices for a TTS provider, optionally filtered by language. - - For ElevenLabs, fetches both the user's own voices and popular shared - voices for the requested language (via the shared-voices library). - """ - if provider == "elevenlabs": - api_key = os.getenv("ELEVENLABS_API_KEY", "") - if not api_key: - return JSONResponse({"error": "ELEVENLABS_API_KEY not set"}, status_code=500) - - def _normalize_voice(v, source="user"): - labels = v.get("labels") or {} - return { - "voice_id": v["voice_id"], - "name": v["name"], - "language": labels.get("language", v.get("language", "")), - "accent": labels.get("accent", v.get("accent", "")), - "gender": labels.get("gender", v.get("gender", "")), - "age": labels.get("age", ""), - "description": labels.get("description", v.get("description", "")), - "use_case": labels.get("use_case", v.get("use_case", "")), - "source": source, - } - - async with httpx.AsyncClient(timeout=30.0) as client: - # Always fetch user's own voices - resp = await client.get( - "https://api.elevenlabs.io/v1/voices", - headers={"xi-api-key": api_key}, - ) - resp.raise_for_status() - user_voices = [ - _normalize_voice(v, "user") - for v in resp.json().get("voices", []) - ] - - # If a language is specified, also fetch shared voices for it - shared_voices = [] - if language: - shared_resp = await client.get( - "https://api.elevenlabs.io/v1/shared-voices", - params={"page_size": 20, "language": language}, - headers={"xi-api-key": api_key}, - ) - if shared_resp.status_code == 200: - shared_voices = [ - _normalize_voice(v, "shared") - for v in shared_resp.json().get("voices", []) - ] - - # Deduplicate (user voices take priority) - seen_ids = {v["voice_id"] for v in user_voices} - combined = user_voices + [v for v in shared_voices if v["voice_id"] not in seen_ids] - return JSONResponse({"voices": combined}) - return JSONResponse({"error": f"Unknown provider: {provider}"}, status_code=400) - - -async def _generate_tts_audio(text: str, tts_model: str, provider: str) -> bytes: - """Route TTS generation to the correct provider.""" - if provider == "elevenlabs": - api_key = os.getenv("ELEVENLABS_API_KEY", "") - if not api_key: - raise ValueError("ELEVENLABS_API_KEY not set") - return await _elevenlabs_tts_generate(text, tts_model, api_key) - else: - api_key = os.getenv("DEEPGRAM_API_KEY", "") - return await _tts_generate(text, tts_model, api_key) - - -@fastapi_app.post("/api/tts-transcribe") -async def tts_transcribe(request: Request): - body = await request.json() - text = body.get("text", "").strip() - tts_model = body.get("tts_model", "aura-2-asteria-en") - tts_provider = body.get("tts_provider", "deepgram") - stt_params = body.get("stt_params", {}) - mode = body.get("mode", "batch") # "batch" | "streaming" | "both" - - if not text: - return JSONResponse({"error": "text is required"}, status_code=400) - if mode not in ("batch", "streaming", "both"): - return JSONResponse({"error": "mode must be batch, streaming, or both"}, status_code=400) - - api_key = os.getenv("DEEPGRAM_API_KEY", "") - - try: - if mode == "batch": - audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) - result = await _stt_batch(audio_bytes, stt_params, api_key) - return JSONResponse(result) - - elif mode == "streaming": - if tts_provider == "elevenlabs": - # ElevenLabs doesn't integrate with Deepgram streaming pipe, - # so generate full audio first, then stream it to STT - audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) - result = await _stt_batch(audio_bytes, stt_params, api_key) - return JSONResponse(result) - result = await _stt_streaming(text, tts_model, stt_params, api_key) - return JSONResponse(result) - - else: # both - if tts_provider == "elevenlabs": - # Can't do streaming pipe with ElevenLabs, run batch only - audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) - result = await _stt_batch(audio_bytes, stt_params, api_key) - return JSONResponse(result) - - async def _batch_pipeline(): - audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) - return await _stt_batch(audio_bytes, stt_params, api_key) - - batch_result, stream_result = await asyncio.gather( - _batch_pipeline(), - _stt_streaming(text, tts_model, stt_params, api_key), - ) - return JSONResponse({"batch": batch_result, "streaming": stream_result}) - - except httpx.HTTPStatusError as e: - return JSONResponse({"error": str(e)}, status_code=e.response.status_code) - except Exception as e: - return JSONResponse({"error": _clean_error(e)}, status_code=500) - - -@fastapi_app.post("/transcribe") -async def transcribe(request: Request): - body = await request.json() - params = body.get("params", {}) - url = body.get("url") - filename = body.get("filename") - - if not url and not filename: - return JSONResponse({"error": "url or filename required"}, status_code=400) - - api_key = os.getenv("DEEPGRAM_API_KEY", "") - base_url = params.get("base_url", "api.deepgram.com") - - # Build clean query params for batch mode, convert bools to lowercase strings - clean = clean_params(params, Mode.BATCH) - query_params = {} - for k, v in clean.items(): - if isinstance(v, bool): - query_params[k] = "true" if v else "false" - elif isinstance(v, (list, str)): - query_params[k] = v - else: - query_params[k] = str(v) - query_params.setdefault("model", "nova-2") - - headers = {"Authorization": f"Token {api_key}"} - listen_url = f"https://{base_url}/v1/listen" - - try: - async with httpx.AsyncClient(timeout=300.0) as client: - if url: - resp = await client.post( - listen_url, - headers={**headers, "Content-Type": "application/json"}, - params=query_params, - json={"url": url}, - ) - else: - file_path = TEMP_DIR / filename - if not file_path.exists(): - return JSONResponse({"error": "File not found"}, status_code=404) - file_bytes = file_path.read_bytes() - resp = await client.post( - listen_url, - headers={**headers, "Content-Type": "audio/*"}, - params=query_params, - content=file_bytes, - ) - resp.raise_for_status() - return JSONResponse(resp.json()) - except httpx.HTTPStatusError as e: - return JSONResponse({"error": str(e)}, status_code=e.response.status_code) - except Exception as e: - return JSONResponse({"error": str(e)}, status_code=500) - - -# --- Helper functions --- - -def _params_to_sdk_kwargs(raw_params: dict) -> dict: - """Convert frontend params dict to deepgram-sdk 6.x keyword args. - model is required by connect() — default to nova-2 if not provided. - """ - clean = clean_params(raw_params, Mode.STREAMING) - kwargs = {} - for k, v in clean.items(): - if isinstance(v, bool): - kwargs[k] = "true" if v else "false" - elif isinstance(v, (list, str)): - kwargs[k] = v - else: - kwargs[k] = str(v) - kwargs.setdefault("model", "nova-2") - return kwargs - - -# --- Streaming Task --- - -async def streaming_task(sid: str, params: dict, stop_event: asyncio.Event) -> None: - """Owns the Deepgram WebSocket lifecycle for one SocketIO session. - Runs as an asyncio.Task. Emits stream_started, transcription_update, stream_finished. - """ - api_key = os.getenv("DEEPGRAM_API_KEY", "") - dg = AsyncDeepgramClient(api_key=api_key) - sdk_kwargs = _params_to_sdk_kwargs(params) - - try: - async with dg.listen.v1.connect(**sdk_kwargs) as ws: - # Store ws so on_audio_stream can call ws.send_media() - if sid in _sessions: - _sessions[sid]["ws"] = ws - - async def on_message(msg, **kwargs): - logger.debug("[%s] on_message type=%s", sid, type(msg).__name__) - if isinstance(msg, ListenV1Metadata): - if sid in _sessions: - _sessions[sid]["request_id"] = msg.request_id - elif isinstance(msg, ListenV1Results): - transcript = msg.channel.alternatives[0].transcript - is_final = bool(msg.is_final) - await sio.emit("transcription_update", { - "transcript": transcript, - "is_final": is_final, - }, to=sid) - - ws.on(EventType.MESSAGE, on_message) - listen_task = asyncio.create_task(ws.start_listening()) - - # Emit stream_started immediately — don't gate on Metadata arrival - await sio.emit("stream_started", {"request_id": None}, to=sid) - - # Keep-alive loop — sends every 8s (under Deepgram's ~10s idle timeout) - async def keep_alive_loop(): - while not stop_event.is_set(): - await asyncio.sleep(8) - if not stop_event.is_set(): - try: - await ws.send_keep_alive() - except Exception as e: - logger.warning("[%s] keep_alive error: %s", sid, e) - break - - ka_task = asyncio.create_task(keep_alive_loop()) - - # Wait for stop signal from on_toggle_transcription(stop) or disconnect() - await stop_event.wait() - - # Graceful shutdown: cancel keep-alive, send CloseStream, await final results - ka_task.cancel() - try: - await ws.send_close_stream() - await listen_task # blocks until Deepgram flushes final Results + closes - except (asyncio.CancelledError, Exception) as e: - logger.warning("[%s] Error during graceful shutdown: %s", sid, e) - if not listen_task.done(): - listen_task.cancel() - - except Exception as e: - logger.error("[%s] streaming_task error: %s", sid, e) - _sessions.pop(sid, None) # Free slot before notifying client so retries aren't blocked - await sio.emit("stream_error", {"message": _clean_error(e)}, to=sid) - finally: - request_id = _sessions[sid].get("request_id") if sid in _sessions else None - await sio.emit("stream_finished", {"request_id": request_id}, to=sid) - _sessions.pop(sid, None) - logger.info("[%s] streaming_task finished, session cleaned up", sid) - - -# --- File Streaming Task --- - -CHUNK_SIZE = 4096 - - -async def file_streaming_task( - sid: str, filename: str, params: dict, stop_event: asyncio.Event -) -> None: - """Streams an uploaded file to Deepgram over WebSocket. - Mirrors streaming_task() but reads from a local file instead of waiting on stop_event. - Emits stream_started, transcription_update, stream_finished. - """ - api_key = os.getenv("DEEPGRAM_API_KEY", "") - dg = AsyncDeepgramClient(api_key=api_key) - sdk_kwargs = _params_to_sdk_kwargs(params) - file_path = TEMP_DIR / filename - - try: - async with dg.listen.v1.connect(**sdk_kwargs) as ws: - # Store ws reference in session - if sid in _sessions: - _sessions[sid]["ws"] = ws - - async def on_message(msg, **kwargs): - logger.debug("[%s] file on_message type=%s", sid, type(msg).__name__) - if isinstance(msg, ListenV1Metadata): - if sid in _sessions: - _sessions[sid]["request_id"] = msg.request_id - elif isinstance(msg, ListenV1Results): - transcript = msg.channel.alternatives[0].transcript - is_final = bool(msg.is_final) - await sio.emit("transcription_update", { - "transcript": transcript, - "is_final": is_final, - "start": msg.start, - }, to=sid) - - ws.on(EventType.MESSAGE, on_message) - listen_task = asyncio.create_task(ws.start_listening()) - - # Emit stream_started immediately — same pattern as streaming_task - await sio.emit("stream_started", {"request_id": None}, to=sid) - - # Compute real-time pacing: sleep between chunks so Deepgram - # receives audio at 1x speed, keeping transcripts in sync with playback. - try: - audio_info = MutagenFile(file_path) - duration = audio_info.info.length if audio_info else None - except Exception: - duration = None - file_size = file_path.stat().st_size - sleep_per_chunk = (CHUNK_SIZE / file_size * duration) if duration and file_size else 0 - - # Stream file in chunks; stop early if stop_event set - try: - with open(file_path, "rb") as f: - while not stop_event.is_set(): - chunk = f.read(CHUNK_SIZE) - if not chunk: - break - await ws.send_media(chunk) - if sleep_per_chunk: - await asyncio.sleep(sleep_per_chunk) - except FileNotFoundError: - await sio.emit("stream_error", {"message": f"File not found: {filename}"}, to=sid) - # Graceful shutdown even on FileNotFoundError - try: - await ws.send_close_stream() - await listen_task - except (asyncio.CancelledError, Exception) as e: - logger.warning("[%s] Error during file-not-found shutdown: %s", sid, e) - if not listen_task.done(): - listen_task.cancel() - return - - # EOF reached (or stop_event set) — flush final words (STR-04 pattern) - try: - await ws.send_close_stream() - await listen_task # blocks until Deepgram flushes final Results + closes - except (asyncio.CancelledError, Exception) as e: - logger.warning("[%s] Error during file streaming graceful shutdown: %s", sid, e) - if not listen_task.done(): - listen_task.cancel() - - except Exception as e: - logger.error("[%s] file_streaming_task error: %s", sid, e) - _sessions.pop(sid, None) - await sio.emit("stream_error", {"message": _clean_error(e)}, to=sid) - finally: - request_id = _sessions[sid].get("request_id") if sid in _sessions else None - await sio.emit("stream_finished", {"request_id": request_id}, to=sid) - _sessions.pop(sid, None) - logger.info("[%s] file_streaming_task finished, session cleaned up", sid) - - -# --- SocketIO Event Handlers --- - -@sio.event -async def connect(sid, environ, auth=None): - logger.info("Client connected: %s", sid) - - -@sio.event -async def disconnect(sid, reason=None): - session = _sessions.pop(sid, None) - if session: - session["stop_event"].set() - task = session.get("task") - if task and not task.done(): - task.cancel() - logger.info("Client disconnected: %s reason=%s", sid, reason) - - -@sio.on("toggle_transcription") -async def on_toggle_transcription(sid, data): - action = data.get("action", "start") - params = data.get("params", data.get("config", {})) - logger.info("[%s] toggle_transcription action=%s", sid, action) - - if action == "start": - if sid in _sessions: - logger.warning("[%s] toggle_transcription(start) while already streaming — ignoring", sid) - return - stop_event = asyncio.Event() - _sessions[sid] = {"stop_event": stop_event, "ws": None, "request_id": None} - task = asyncio.create_task(streaming_task(sid, params, stop_event)) - _sessions[sid]["task"] = task - - elif action == "stop": - if sid not in _sessions: - # Not streaming — keep frontend in sync - await sio.emit("stream_finished", {"request_id": None}, to=sid) - return - _sessions[sid]["stop_event"].set() - # stream_finished is emitted by streaming_task after listen_task completes - - -@sio.on("audio_stream") -async def on_audio_stream(sid, data): - session = _sessions.get(sid) - if session and session.get("ws") is not None: - try: - audio = data if isinstance(data, bytes) else bytes(data) - await session["ws"].send_media(audio) - except Exception as e: - logger.warning("[%s] send_media error: %s", sid, e) - # If ws is None (WebSocket not yet open), drop silently — browser buffers more audio - - -@sio.on("detect_audio_settings") -async def on_detect_audio_settings(sid): - try: - from common.audio_settings import detect_audio_settings - settings = detect_audio_settings() - await sio.emit("audio_settings", { - "sample_rate": int(settings.get("sample_rate", 16000)), - "channels": int(settings.get("max_input_channels", 1)), - }, to=sid) - except Exception as e: - logger.warning("Audio settings detection failed: %s", e) - await sio.emit("audio_settings", {"sample_rate": 16000, "channels": 1}, to=sid) - - -@sio.on("start_file_streaming") -async def on_start_file_streaming(sid, data): - filename = data.get("filename") if data else None - params = data.get("params", {}) if data else {} - logger.info("[%s] start_file_streaming filename=%s", sid, filename) - - if not filename: - await sio.emit("stream_error", {"message": "filename is required"}, to=sid) - return - - if sid in _sessions: - logger.warning("[%s] start_file_streaming while already streaming — ignoring", sid) - return - - stop_event = asyncio.Event() - _sessions[sid] = {"stop_event": stop_event, "ws": None, "request_id": None} - task = asyncio.create_task(file_streaming_task(sid, filename, params, stop_event)) - _sessions[sid]["task"] = task - - -@sio.on("stop_file_streaming") -async def on_stop_file_streaming(sid, data=None): - logger.info("[%s] stop_file_streaming", sid) - - if sid not in _sessions: - # Not streaming — keep frontend in sync - await sio.emit("stream_finished", {"request_id": None}, to=sid) - return - - _sessions[sid]["stop_event"].set() - # stream_finished is emitted by file_streaming_task after listen_task completes +import asyncio +import json as json_mod +import logging +import os +import re +import tempfile +import urllib.parse +from pathlib import Path + +from mutagen import File as MutagenFile + +import httpx +import socketio +import websockets +from deepgram import AsyncDeepgramClient +from deepgram.core.events import EventType +from deepgram.listen.v1.types import ListenV1Results, ListenV1Metadata +from dotenv import load_dotenv +from fastapi import FastAPI, Request, UploadFile, File +from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles + +from stt.options import clean_params, Mode + +load_dotenv() + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +PORT = int(os.getenv("PORT", 8001)) +TEMP_DIR = Path(tempfile.gettempdir()) / "deepgram-stt" +TEMP_DIR.mkdir(exist_ok=True) + +# --- Security: outbound destination allowlist --- +# DO NOT interpolate a caller-supplied base_url into an outbound URL that +# carries DEEPGRAM_API_KEY. A free-form base_url let any caller choose the +# destination host AND inject path/query, which forwarded this server's API +# key to a host of their choosing. Verified exploitable 2026-07-25: a request +# with base_url="postman-echo.com/post?x=" delivered "Token " to that +# third party. The allowlist is operator-controlled via env, never caller +# controlled, and the host must be a BARE hostname with optional port so no +# path, query, fragment, or userinfo can be smuggled in. +DEEPGRAM_HOST = "api.deepgram.com" +ALLOWED_STT_HOSTS = { + h.strip().lower() + for h in os.getenv("ALLOWED_STT_HOSTS", DEEPGRAM_HOST).split(",") + if h.strip() +} +_BARE_HOST_RE = re.compile(r"^[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:\d{1,5})?$") + +# Caps so an unauthenticated caller cannot run up an unbounded bill or fill the +# disk. TTS is billed per character and STT per audio-minute, so the text cap is +# a spend cap, not just a validation nicety. +MAX_TTS_CHARS = int(os.getenv("MAX_TTS_CHARS", 2000)) +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", 100 * 1024 * 1024)) + + +class HostNotAllowed(ValueError): + """Caller-supplied base_url is malformed or not in ALLOWED_STT_HOSTS.""" + + +def _resolve_stt_host(params: dict) -> str: + """Validate a caller-supplied base_url against the operator allowlist. + + Returns a bare host safe to interpolate into an outbound URL that carries + the server's Deepgram credential. Raises HostNotAllowed otherwise. + """ + raw = (params.get("base_url") or DEEPGRAM_HOST).strip().lower() + if not _BARE_HOST_RE.match(raw): + raise HostNotAllowed( + "base_url must be a bare hostname with optional port, " + "no scheme, path, query, or credentials" + ) + if raw not in ALLOWED_STT_HOSTS: + raise HostNotAllowed(f"base_url host is not allowlisted: {raw}") + return raw + + +def _safe_temp_path(filename: str) -> Path: + """Resolve an upload filename to a path guaranteed to sit inside TEMP_DIR. + + DO NOT join a caller-supplied filename onto TEMP_DIR directly. Python's + Path join REPLACES the base when the right side is absolute, so + TEMP_DIR / "/etc/passwd" is "/etc/passwd", and "../" components traverse + out. Both were verified exploitable as arbitrary file writes 2026-07-25. + """ + name = Path(filename or "").name + if not name or name in (".", "..") or name.startswith("."): + raise ValueError("invalid filename") + path = (TEMP_DIR / name).resolve() + if path.parent != TEMP_DIR.resolve(): + raise ValueError("resolved path escapes the temp directory") + return path + +# 1. AsyncServer — async_mode MUST be "asgi" (not "gevent", not "threading") +sio = socketio.AsyncServer( + async_mode="asgi", + cors_allowed_origins="*", +) + +# 2. FastAPI sub-app for HTTP routes only +fastapi_app = FastAPI() +fastapi_app.mount("/static", StaticFiles(directory="static"), name="static") + +# 3. Combined ASGI callable — THIS is what uvicorn serves, not fastapi_app +app = socketio.ASGIApp(sio, fastapi_app) + +def _clean_error(e: Exception) -> str: + """Strip SDK request headers (including auth token) from Deepgram exception messages.""" + msg = str(e) + m = re.search(r'status_code:\s*(\d+),\s*body:\s*(.+)$', msg, re.DOTALL) + if m: + return f"Deepgram {m.group(1)}: {m.group(2).strip()}" + return msg + + +# Per-session state — module-level dict, not sio.session() (too slow for audio hot path) +# Key: SocketIO session id (sid) +# Value: dict with keys: task (asyncio.Task), stop_event (asyncio.Event), +# ws (AsyncV1SocketClient | None), request_id (str | None) +_sessions: dict[str, dict] = {} + + +# --- HTTP Routes --- + +@fastapi_app.get("/") +async def index(): + return FileResponse("templates/index.html") + + +@fastapi_app.post("/upload") +async def upload(file: UploadFile = File(...)): + try: + path = _safe_temp_path(file.filename) + except ValueError as e: + return JSONResponse({"error": str(e)}, status_code=400) + content = await file.read() + if len(content) > MAX_UPLOAD_BYTES: + return JSONResponse( + {"error": f"file exceeds {MAX_UPLOAD_BYTES} bytes"}, status_code=413 + ) + path.write_bytes(content) + # Report the sanitized name; the caller must use it for follow-up calls. + return JSONResponse({"filename": path.name, "size": path.stat().st_size}) + + +@fastapi_app.get("/files/{filename}") +async def serve_file(filename: str): + try: + path = _safe_temp_path(filename) + except ValueError as e: + return JSONResponse({"error": str(e)}, status_code=400) + if not path.exists(): + return JSONResponse({"error": "not found"}, status_code=404) + return FileResponse(path) + + +async def _tts_generate(text: str, tts_model: str, api_key: str) -> bytes: + """Call Deepgram TTS and return MP3 bytes.""" + headers = {"Authorization": f"Token {api_key}"} + async with httpx.AsyncClient(timeout=60.0) as client: + resp = await client.post( + "https://api.deepgram.com/v1/speak", + headers={**headers, "Content-Type": "application/json"}, + params={"model": tts_model, "encoding": "mp3"}, + json={"text": text}, + ) + resp.raise_for_status() + return resp.content + + +async def _elevenlabs_tts_generate(text: str, voice_id: str, api_key: str) -> bytes: + """Call ElevenLabs TTS and return MP3 bytes.""" + async with httpx.AsyncClient(timeout=60.0) as client: + resp = await client.post( + f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}", + headers={ + "xi-api-key": api_key, + "Content-Type": "application/json", + "Accept": "audio/mpeg", + }, + json={ + "text": text, + "model_id": "eleven_multilingual_v2", + }, + ) + resp.raise_for_status() + return resp.content + + +async def _stt_batch(audio_bytes: bytes, stt_params: dict, api_key: str) -> dict: + """Transcribe audio bytes via Deepgram pre-recorded (batch) API.""" + headers = {"Authorization": f"Token {api_key}"} + base_url = _resolve_stt_host(stt_params) + clean = clean_params(stt_params, Mode.BATCH) + query_params = {} + for k, v in clean.items(): + if isinstance(v, bool): + query_params[k] = "true" if v else "false" + elif isinstance(v, (list, str)): + query_params[k] = v + else: + query_params[k] = str(v) + query_params.setdefault("model", "nova-2") + + async with httpx.AsyncClient(timeout=60.0) as client: + resp = await client.post( + f"https://{base_url}/v1/listen", + headers={**headers, "Content-Type": "audio/mp3"}, + params=query_params, + content=audio_bytes, + ) + resp.raise_for_status() + return resp.json() + + +async def _stt_streaming(text: str, tts_model: str, stt_params: dict, api_key: str) -> dict: + """Pipe Deepgram TTS streaming response directly into the STT WebSocket. + TTS audio chunks are forwarded to STT as they arrive — naturally paced at + speech speed, no buffering or artificial timing needed. + Returns {"transcript": str, "segments": list, "raw_responses": list}. + """ + base_url = _resolve_stt_host(stt_params) + headers = {"Authorization": f"Token {api_key}"} + + # For custom endpoints (e.g. aiworks), use raw websockets instead of the SDK + if base_url != DEEPGRAM_HOST: + return await _stt_streaming_raw(text, tts_model, stt_params, api_key) + + dg = AsyncDeepgramClient(api_key=api_key) + sdk_kwargs = _params_to_sdk_kwargs(stt_params) + + segments = [] + + async with dg.listen.v1.connect(**sdk_kwargs) as ws: + async def on_message(msg, **kwargs): + if isinstance(msg, ListenV1Results) and bool(msg.is_final): + t = msg.channel.alternatives[0].transcript + if t.strip(): + segments.append(t) + + ws.on(EventType.MESSAGE, on_message) + listen_task = asyncio.create_task(ws.start_listening()) + + # Stream TTS audio directly into STT WebSocket as chunks arrive + async with httpx.AsyncClient(timeout=60.0) as client: + async with client.stream( + "POST", + "https://api.deepgram.com/v1/speak", + headers={**headers, "Content-Type": "application/json"}, + params={"model": tts_model, "encoding": "mp3"}, + json={"text": text}, + ) as tts_resp: + tts_resp.raise_for_status() + async for chunk in tts_resp.aiter_bytes(chunk_size=4096): + await ws.send_media(chunk) + + await ws.send_close_stream() + await listen_task + + return { + "transcript": " ".join(segments), + "segments": segments, + } + + +async def _stt_streaming_raw(text: str, tts_model: str, stt_params: dict, api_key: str) -> dict: + """Stream TTS audio into a custom WebSocket STT endpoint (e.g. aiworks). + Uses raw websockets since the Deepgram SDK only connects to api.deepgram.com. + Returns full raw responses so callers can inspect the response schema. + """ + base_url = _resolve_stt_host(stt_params) + clean = clean_params(stt_params, Mode.STREAMING) + query_parts = [] + for k, v in clean.items(): + if isinstance(v, bool): + query_parts.append(f"{k}={'true' if v else 'false'}") + elif isinstance(v, list): + for item in v: + query_parts.append(f"{k}={urllib.parse.quote(str(item))}") + else: + query_parts.append(f"{k}={urllib.parse.quote(str(v))}") + qs = "&".join(query_parts) + ws_url = f"wss://{base_url}/v1/listen?{qs}" if qs else f"wss://{base_url}/v1/listen" + headers = {"Authorization": f"Token {api_key}"} + + # Generate TTS audio first (full buffer), then stream into WebSocket + async with httpx.AsyncClient(timeout=60.0) as client: + tts_resp = await client.post( + "https://api.deepgram.com/v1/speak", + headers={**headers, "Content-Type": "application/json"}, + params={"model": tts_model, "encoding": "mp3"}, + json={"text": text}, + ) + tts_resp.raise_for_status() + audio_bytes = tts_resp.content + + segments = [] + raw_responses = [] + + async with websockets.connect(ws_url, additional_headers=headers) as ws: + # Send audio in chunks + chunk_size = 4096 + for i in range(0, len(audio_bytes), chunk_size): + await ws.send(audio_bytes[i:i + chunk_size]) + await asyncio.sleep(0.01) + + # Signal end of audio + await ws.send(json_mod.dumps({"type": "CloseStream"})) + + # Collect all responses until connection closes + try: + async for msg in ws: + if isinstance(msg, str): + data = json_mod.loads(msg) + raw_responses.append(data) + # Extract transcript from various response shapes + if "deepgram_stt" in data: + stt = data["deepgram_stt"] + if isinstance(stt, list): + stt = stt[0] + t = stt.get("channel", {}).get("alternatives", [{}])[0].get("transcript", "") + is_final = stt.get("is_final", False) + if t.strip() and is_final: + segments.append(t) + elif data.get("type") == "Results": + t = data.get("channel", {}).get("alternatives", [{}])[0].get("transcript", "") + if t.strip() and data.get("is_final"): + segments.append(t) + elif data.get("type") == "text": + t = data.get("data", "") + if t.strip(): + segments.append(t) + # Top-level transcript (aiworks wrapper) + elif "transcript" in data and not any(k in data for k in ("deepgram_stt", "type")): + t = data["transcript"] + if t.strip(): + segments.append(t) + except websockets.exceptions.ConnectionClosed: + pass + + return { + "transcript": " ".join(segments), + "segments": segments, + "raw_responses": raw_responses, + } + + +@fastapi_app.get("/api/tts-voices") +async def tts_voices(provider: str = "elevenlabs", language: str = ""): + """Return available voices for a TTS provider, optionally filtered by language. + + For ElevenLabs, fetches both the user's own voices and popular shared + voices for the requested language (via the shared-voices library). + """ + if provider == "elevenlabs": + api_key = os.getenv("ELEVENLABS_API_KEY", "") + if not api_key: + return JSONResponse({"error": "ELEVENLABS_API_KEY not set"}, status_code=500) + + def _normalize_voice(v, source="user"): + labels = v.get("labels") or {} + return { + "voice_id": v["voice_id"], + "name": v["name"], + "language": labels.get("language", v.get("language", "")), + "accent": labels.get("accent", v.get("accent", "")), + "gender": labels.get("gender", v.get("gender", "")), + "age": labels.get("age", ""), + "description": labels.get("description", v.get("description", "")), + "use_case": labels.get("use_case", v.get("use_case", "")), + "source": source, + } + + async with httpx.AsyncClient(timeout=30.0) as client: + # Always fetch user's own voices + resp = await client.get( + "https://api.elevenlabs.io/v1/voices", + headers={"xi-api-key": api_key}, + ) + resp.raise_for_status() + user_voices = [ + _normalize_voice(v, "user") + for v in resp.json().get("voices", []) + ] + + # If a language is specified, also fetch shared voices for it + shared_voices = [] + if language: + shared_resp = await client.get( + "https://api.elevenlabs.io/v1/shared-voices", + params={"page_size": 20, "language": language}, + headers={"xi-api-key": api_key}, + ) + if shared_resp.status_code == 200: + shared_voices = [ + _normalize_voice(v, "shared") + for v in shared_resp.json().get("voices", []) + ] + + # Deduplicate (user voices take priority) + seen_ids = {v["voice_id"] for v in user_voices} + combined = user_voices + [v for v in shared_voices if v["voice_id"] not in seen_ids] + return JSONResponse({"voices": combined}) + return JSONResponse({"error": f"Unknown provider: {provider}"}, status_code=400) + + +async def _generate_tts_audio(text: str, tts_model: str, provider: str) -> bytes: + """Route TTS generation to the correct provider.""" + if provider == "elevenlabs": + api_key = os.getenv("ELEVENLABS_API_KEY", "") + if not api_key: + raise ValueError("ELEVENLABS_API_KEY not set") + return await _elevenlabs_tts_generate(text, tts_model, api_key) + else: + api_key = os.getenv("DEEPGRAM_API_KEY", "") + return await _tts_generate(text, tts_model, api_key) + + +@fastapi_app.post("/api/tts-transcribe") +async def tts_transcribe(request: Request): + body = await request.json() + text = body.get("text", "").strip() + tts_model = body.get("tts_model", "aura-2-asteria-en") + tts_provider = body.get("tts_provider", "deepgram") + stt_params = body.get("stt_params", {}) + mode = body.get("mode", "batch") # "batch" | "streaming" | "both" + + if not text: + return JSONResponse({"error": "text is required"}, status_code=400) + if len(text) > MAX_TTS_CHARS: + return JSONResponse( + {"error": f"text exceeds {MAX_TTS_CHARS} characters"}, status_code=413 + ) + if mode not in ("batch", "streaming", "both"): + return JSONResponse({"error": "mode must be batch, streaming, or both"}, status_code=400) + try: + _resolve_stt_host(stt_params) + except HostNotAllowed as e: + return JSONResponse({"error": str(e)}, status_code=400) + + api_key = os.getenv("DEEPGRAM_API_KEY", "") + + try: + if mode == "batch": + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + result = await _stt_batch(audio_bytes, stt_params, api_key) + return JSONResponse(result) + + elif mode == "streaming": + if tts_provider == "elevenlabs": + # ElevenLabs doesn't integrate with Deepgram streaming pipe, + # so generate full audio first, then stream it to STT + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + result = await _stt_batch(audio_bytes, stt_params, api_key) + return JSONResponse(result) + result = await _stt_streaming(text, tts_model, stt_params, api_key) + return JSONResponse(result) + + else: # both + if tts_provider == "elevenlabs": + # Can't do streaming pipe with ElevenLabs, run batch only + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + result = await _stt_batch(audio_bytes, stt_params, api_key) + return JSONResponse(result) + + async def _batch_pipeline(): + audio_bytes = await _generate_tts_audio(text, tts_model, tts_provider) + return await _stt_batch(audio_bytes, stt_params, api_key) + + batch_result, stream_result = await asyncio.gather( + _batch_pipeline(), + _stt_streaming(text, tts_model, stt_params, api_key), + ) + return JSONResponse({"batch": batch_result, "streaming": stream_result}) + + except httpx.HTTPStatusError as e: + return JSONResponse({"error": str(e)}, status_code=e.response.status_code) + except Exception as e: + return JSONResponse({"error": _clean_error(e)}, status_code=500) + + +@fastapi_app.post("/transcribe") +async def transcribe(request: Request): + body = await request.json() + params = body.get("params", {}) + url = body.get("url") + filename = body.get("filename") + + if not url and not filename: + return JSONResponse({"error": "url or filename required"}, status_code=400) + + api_key = os.getenv("DEEPGRAM_API_KEY", "") + try: + base_url = _resolve_stt_host(params) + except HostNotAllowed as e: + return JSONResponse({"error": str(e)}, status_code=400) + + # Build clean query params for batch mode, convert bools to lowercase strings + clean = clean_params(params, Mode.BATCH) + query_params = {} + for k, v in clean.items(): + if isinstance(v, bool): + query_params[k] = "true" if v else "false" + elif isinstance(v, (list, str)): + query_params[k] = v + else: + query_params[k] = str(v) + query_params.setdefault("model", "nova-2") + + headers = {"Authorization": f"Token {api_key}"} + listen_url = f"https://{base_url}/v1/listen" + + try: + async with httpx.AsyncClient(timeout=300.0) as client: + if url: + resp = await client.post( + listen_url, + headers={**headers, "Content-Type": "application/json"}, + params=query_params, + json={"url": url}, + ) + else: + file_path = TEMP_DIR / filename + if not file_path.exists(): + return JSONResponse({"error": "File not found"}, status_code=404) + file_bytes = file_path.read_bytes() + resp = await client.post( + listen_url, + headers={**headers, "Content-Type": "audio/*"}, + params=query_params, + content=file_bytes, + ) + resp.raise_for_status() + return JSONResponse(resp.json()) + except httpx.HTTPStatusError as e: + return JSONResponse({"error": _clean_error(e)}, status_code=e.response.status_code) + except Exception as e: + return JSONResponse({"error": _clean_error(e)}, status_code=500) + + +# --- Helper functions --- + +def _params_to_sdk_kwargs(raw_params: dict) -> dict: + """Convert frontend params dict to deepgram-sdk 6.x keyword args. + model is required by connect() — default to nova-2 if not provided. + """ + clean = clean_params(raw_params, Mode.STREAMING) + kwargs = {} + for k, v in clean.items(): + if isinstance(v, bool): + kwargs[k] = "true" if v else "false" + elif isinstance(v, (list, str)): + kwargs[k] = v + else: + kwargs[k] = str(v) + kwargs.setdefault("model", "nova-2") + return kwargs + + +# --- Streaming Task --- + +async def streaming_task(sid: str, params: dict, stop_event: asyncio.Event) -> None: + """Owns the Deepgram WebSocket lifecycle for one SocketIO session. + Runs as an asyncio.Task. Emits stream_started, transcription_update, stream_finished. + """ + api_key = os.getenv("DEEPGRAM_API_KEY", "") + dg = AsyncDeepgramClient(api_key=api_key) + sdk_kwargs = _params_to_sdk_kwargs(params) + + try: + async with dg.listen.v1.connect(**sdk_kwargs) as ws: + # Store ws so on_audio_stream can call ws.send_media() + if sid in _sessions: + _sessions[sid]["ws"] = ws + + async def on_message(msg, **kwargs): + logger.debug("[%s] on_message type=%s", sid, type(msg).__name__) + if isinstance(msg, ListenV1Metadata): + if sid in _sessions: + _sessions[sid]["request_id"] = msg.request_id + elif isinstance(msg, ListenV1Results): + transcript = msg.channel.alternatives[0].transcript + is_final = bool(msg.is_final) + await sio.emit("transcription_update", { + "transcript": transcript, + "is_final": is_final, + }, to=sid) + + ws.on(EventType.MESSAGE, on_message) + listen_task = asyncio.create_task(ws.start_listening()) + + # Emit stream_started immediately — don't gate on Metadata arrival + await sio.emit("stream_started", {"request_id": None}, to=sid) + + # Keep-alive loop — sends every 8s (under Deepgram's ~10s idle timeout) + async def keep_alive_loop(): + while not stop_event.is_set(): + await asyncio.sleep(8) + if not stop_event.is_set(): + try: + await ws.send_keep_alive() + except Exception as e: + logger.warning("[%s] keep_alive error: %s", sid, e) + break + + ka_task = asyncio.create_task(keep_alive_loop()) + + # Wait for stop signal from on_toggle_transcription(stop) or disconnect() + await stop_event.wait() + + # Graceful shutdown: cancel keep-alive, send CloseStream, await final results + ka_task.cancel() + try: + await ws.send_close_stream() + await listen_task # blocks until Deepgram flushes final Results + closes + except (asyncio.CancelledError, Exception) as e: + logger.warning("[%s] Error during graceful shutdown: %s", sid, e) + if not listen_task.done(): + listen_task.cancel() + + except Exception as e: + logger.error("[%s] streaming_task error: %s", sid, e) + _sessions.pop(sid, None) # Free slot before notifying client so retries aren't blocked + await sio.emit("stream_error", {"message": _clean_error(e)}, to=sid) + finally: + request_id = _sessions[sid].get("request_id") if sid in _sessions else None + await sio.emit("stream_finished", {"request_id": request_id}, to=sid) + _sessions.pop(sid, None) + logger.info("[%s] streaming_task finished, session cleaned up", sid) + + +# --- File Streaming Task --- + +CHUNK_SIZE = 4096 + + +async def file_streaming_task( + sid: str, filename: str, params: dict, stop_event: asyncio.Event +) -> None: + """Streams an uploaded file to Deepgram over WebSocket. + Mirrors streaming_task() but reads from a local file instead of waiting on stop_event. + Emits stream_started, transcription_update, stream_finished. + """ + api_key = os.getenv("DEEPGRAM_API_KEY", "") + dg = AsyncDeepgramClient(api_key=api_key) + sdk_kwargs = _params_to_sdk_kwargs(params) + file_path = TEMP_DIR / filename + + try: + async with dg.listen.v1.connect(**sdk_kwargs) as ws: + # Store ws reference in session + if sid in _sessions: + _sessions[sid]["ws"] = ws + + async def on_message(msg, **kwargs): + logger.debug("[%s] file on_message type=%s", sid, type(msg).__name__) + if isinstance(msg, ListenV1Metadata): + if sid in _sessions: + _sessions[sid]["request_id"] = msg.request_id + elif isinstance(msg, ListenV1Results): + transcript = msg.channel.alternatives[0].transcript + is_final = bool(msg.is_final) + await sio.emit("transcription_update", { + "transcript": transcript, + "is_final": is_final, + "start": msg.start, + }, to=sid) + + ws.on(EventType.MESSAGE, on_message) + listen_task = asyncio.create_task(ws.start_listening()) + + # Emit stream_started immediately — same pattern as streaming_task + await sio.emit("stream_started", {"request_id": None}, to=sid) + + # Compute real-time pacing: sleep between chunks so Deepgram + # receives audio at 1x speed, keeping transcripts in sync with playback. + try: + audio_info = MutagenFile(file_path) + duration = audio_info.info.length if audio_info else None + except Exception: + duration = None + file_size = file_path.stat().st_size + sleep_per_chunk = (CHUNK_SIZE / file_size * duration) if duration and file_size else 0 + + # Stream file in chunks; stop early if stop_event set + try: + with open(file_path, "rb") as f: + while not stop_event.is_set(): + chunk = f.read(CHUNK_SIZE) + if not chunk: + break + await ws.send_media(chunk) + if sleep_per_chunk: + await asyncio.sleep(sleep_per_chunk) + except FileNotFoundError: + await sio.emit("stream_error", {"message": f"File not found: {filename}"}, to=sid) + # Graceful shutdown even on FileNotFoundError + try: + await ws.send_close_stream() + await listen_task + except (asyncio.CancelledError, Exception) as e: + logger.warning("[%s] Error during file-not-found shutdown: %s", sid, e) + if not listen_task.done(): + listen_task.cancel() + return + + # EOF reached (or stop_event set) — flush final words (STR-04 pattern) + try: + await ws.send_close_stream() + await listen_task # blocks until Deepgram flushes final Results + closes + except (asyncio.CancelledError, Exception) as e: + logger.warning("[%s] Error during file streaming graceful shutdown: %s", sid, e) + if not listen_task.done(): + listen_task.cancel() + + except Exception as e: + logger.error("[%s] file_streaming_task error: %s", sid, e) + _sessions.pop(sid, None) + await sio.emit("stream_error", {"message": _clean_error(e)}, to=sid) + finally: + request_id = _sessions[sid].get("request_id") if sid in _sessions else None + await sio.emit("stream_finished", {"request_id": request_id}, to=sid) + _sessions.pop(sid, None) + logger.info("[%s] file_streaming_task finished, session cleaned up", sid) + + +# --- SocketIO Event Handlers --- + +@sio.event +async def connect(sid, environ, auth=None): + logger.info("Client connected: %s", sid) + + +@sio.event +async def disconnect(sid, reason=None): + session = _sessions.pop(sid, None) + if session: + session["stop_event"].set() + task = session.get("task") + if task and not task.done(): + task.cancel() + logger.info("Client disconnected: %s reason=%s", sid, reason) + + +@sio.on("toggle_transcription") +async def on_toggle_transcription(sid, data): + action = data.get("action", "start") + params = data.get("params", data.get("config", {})) + logger.info("[%s] toggle_transcription action=%s", sid, action) + + if action == "start": + if sid in _sessions: + logger.warning("[%s] toggle_transcription(start) while already streaming — ignoring", sid) + return + stop_event = asyncio.Event() + _sessions[sid] = {"stop_event": stop_event, "ws": None, "request_id": None} + task = asyncio.create_task(streaming_task(sid, params, stop_event)) + _sessions[sid]["task"] = task + + elif action == "stop": + if sid not in _sessions: + # Not streaming — keep frontend in sync + await sio.emit("stream_finished", {"request_id": None}, to=sid) + return + _sessions[sid]["stop_event"].set() + # stream_finished is emitted by streaming_task after listen_task completes + + +@sio.on("audio_stream") +async def on_audio_stream(sid, data): + session = _sessions.get(sid) + if session and session.get("ws") is not None: + try: + audio = data if isinstance(data, bytes) else bytes(data) + await session["ws"].send_media(audio) + except Exception as e: + logger.warning("[%s] send_media error: %s", sid, e) + # If ws is None (WebSocket not yet open), drop silently — browser buffers more audio + + +@sio.on("detect_audio_settings") +async def on_detect_audio_settings(sid): + try: + from common.audio_settings import detect_audio_settings + settings = detect_audio_settings() + await sio.emit("audio_settings", { + "sample_rate": int(settings.get("sample_rate", 16000)), + "channels": int(settings.get("max_input_channels", 1)), + }, to=sid) + except Exception as e: + logger.warning("Audio settings detection failed: %s", e) + await sio.emit("audio_settings", {"sample_rate": 16000, "channels": 1}, to=sid) + + +@sio.on("start_file_streaming") +async def on_start_file_streaming(sid, data): + filename = data.get("filename") if data else None + params = data.get("params", {}) if data else {} + logger.info("[%s] start_file_streaming filename=%s", sid, filename) + + if not filename: + await sio.emit("stream_error", {"message": "filename is required"}, to=sid) + return + + if sid in _sessions: + logger.warning("[%s] start_file_streaming while already streaming — ignoring", sid) + return + + stop_event = asyncio.Event() + _sessions[sid] = {"stop_event": stop_event, "ws": None, "request_id": None} + task = asyncio.create_task(file_streaming_task(sid, filename, params, stop_event)) + _sessions[sid]["task"] = task + + +@sio.on("stop_file_streaming") +async def on_stop_file_streaming(sid, data=None): + logger.info("[%s] stop_file_streaming", sid) + + if sid not in _sessions: + # Not streaming — keep frontend in sync + await sio.emit("stream_finished", {"request_id": None}, to=sid) + return + + _sessions[sid]["stop_event"].set() + # stream_finished is emitted by file_streaming_task after listen_task completes diff --git a/stt/options.py b/stt/options.py index c2e1ede..7134f01 100644 --- a/stt/options.py +++ b/stt/options.py @@ -16,6 +16,14 @@ class Mode(str, Enum): # Params that should never be sent to Deepgram (handled by client) INTERNAL_PARAMS = {"base_url"} +# Params a caller must never be able to set, on any endpoint. +# `callback` makes Deepgram POST the finished transcript to a URL of the +# caller's choosing, so on a public unauthenticated app it is a data-exfil +# primitive AND a way to run arbitrary async jobs on the server's account. +# DO NOT move this into INTERNAL_PARAMS: those are stripped because the client +# consumes them, these are stripped because forwarding them is a vulnerability. +DENIED_PARAMS = {"callback"} + def clean_params(params: dict, mode: Mode) -> dict: """ @@ -25,7 +33,7 @@ def clean_params(params: dict, mode: Mode) -> dict: """ result = {} for key, value in params.items(): - if key in INTERNAL_PARAMS: + if key in INTERNAL_PARAMS or key in DENIED_PARAMS: continue if mode == Mode.STREAMING and key in BATCH_ONLY: continue @@ -41,9 +49,15 @@ def clean_params(params: dict, mode: Mode) -> dict: # Handle keyterms: list of "term" or "term:weight" strings -> repeated keyterm= params # (handled by requests library when value is a list) - # Handle extra params: merge into result + # Handle extra params: merge into result. + # Re-apply the deny list here. This merge runs AFTER the filter loop above, + # so without it a caller smuggles a blocked param straight through as + # extra={"callback": "..."} and the deny list above does nothing. if "extra" in result and isinstance(result["extra"], dict): extra = result.pop("extra") - result.update(extra) + result.update({ + k: v for k, v in extra.items() + if k not in DENIED_PARAMS and k not in INTERNAL_PARAMS + }) return result diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..e1d44f0 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,130 @@ +# tests/test_security.py +# Regression tests for the abuse surface of a PUBLIC, unauthenticated app. +# Every test here corresponds to something verified exploitable on 2026-07-25. +# Uses pytest-asyncio auto mode (no @pytest.mark.asyncio needed). +import os +from pathlib import Path + +import pytest + +os.environ.setdefault("DEEPGRAM_API_KEY", "test-key") + +import app +from stt.options import clean_params, Mode + + +# --------------------------------------------------------------------------- +# base_url must never let a caller choose where the server's API key goes. +# Verified exploit: base_url="postman-echo.com/post?x=" delivered +# "Authorization: Token " to that third-party host. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("evil", [ + "postman-echo.com/post?x=", # path + query injection (the live exploit) + "attacker.example", # bare but not allowlisted + "api.deepgram.com@attacker.tld", # userinfo trick + "api.deepgram.com/../attacker", # traversal in the host slot + "attacker.tld#api.deepgram.com", # fragment trick + "attacker.tld?a=api.deepgram.com", + "https://attacker.tld", # scheme smuggling + "api.deepgram.com evil.tld", # whitespace + "api.deepgram.com:1234", # allowlisted name, unexpected port +]) +def test_base_url_rejects_untrusted_destinations(evil): + with pytest.raises(app.HostNotAllowed): + app._resolve_stt_host({"base_url": evil}) + + +def test_base_url_defaults_to_deepgram(): + assert app._resolve_stt_host({}) == "api.deepgram.com" + assert app._resolve_stt_host({"base_url": ""}) == "api.deepgram.com" + assert app._resolve_stt_host({"base_url": " API.DEEPGRAM.COM "}) == "api.deepgram.com" + + +def test_base_url_allows_an_operator_allowlisted_host(monkeypatch): + """The allowlist is operator-controlled via env, never caller-controlled.""" + monkeypatch.setattr(app, "ALLOWED_STT_HOSTS", {"api.deepgram.com", "flow.aiworks.test"}) + assert app._resolve_stt_host({"base_url": "flow.aiworks.test"}) == "flow.aiworks.test" + with pytest.raises(app.HostNotAllowed): + app._resolve_stt_host({"base_url": "other.aiworks.test"}) + + +# --------------------------------------------------------------------------- +# /upload must not write outside TEMP_DIR. +# Verified exploit: both an absolute filename and ../ traversal wrote to an +# arbitrary path, because Path join REPLACES the base on an absolute RHS. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("evil", [ + "/etc/passwd", + "/tmp/escaped.txt", + "../../../../etc/passwd", + "../escaped.txt", + "..", + ".", + "", + None, + ".ssh", + "sub/dir/file.wav", +]) +def test_upload_filename_cannot_escape_temp_dir(evil): + try: + path = app._safe_temp_path(evil) + except ValueError: + return # rejected outright, which is also correct + assert path.parent == app.TEMP_DIR.resolve(), f"{evil!r} escaped to {path}" + + +def test_safe_temp_path_keeps_a_normal_filename(): + assert app._safe_temp_path("sample.wav").name == "sample.wav" + # A directory component is stripped, not rejected, so uploads still work. + assert app._safe_temp_path("sub/dir/sample.wav").name == "sample.wav" + + +# --------------------------------------------------------------------------- +# callback is a server-side exfil primitive: it makes Deepgram POST the +# transcript to a URL the caller picked, on the server's account. +# --------------------------------------------------------------------------- + +def test_callback_is_never_forwarded(): + out = clean_params({"model": "nova-3", "callback": "https://attacker.tld/c"}, Mode.BATCH) + assert "callback" not in out + + +def test_callback_cannot_be_smuggled_through_extra(): + """The extra merge runs after the filter loop, so it needs its own guard.""" + out = clean_params( + {"model": "nova-3", "extra": {"callback": "https://attacker.tld/c"}}, + Mode.BATCH, + ) + assert "callback" not in out + + +def test_base_url_cannot_be_smuggled_through_extra(): + out = clean_params( + {"model": "nova-3", "extra": {"base_url": "attacker.tld"}}, Mode.BATCH + ) + assert "base_url" not in out + + +def test_extra_still_passes_ordinary_params(): + out = clean_params({"model": "nova-3", "extra": {"custom_flag": "1"}}, Mode.BATCH) + assert out["custom_flag"] == "1" + + +# --------------------------------------------------------------------------- +# Spend and disk caps exist, because the app is unauthenticated. +# --------------------------------------------------------------------------- + +def test_abuse_caps_are_configured(): + assert app.MAX_TTS_CHARS > 0 + assert app.MAX_UPLOAD_BYTES > 0 + + +def test_no_api_key_in_static_assets(): + """The Deepgram key is server-side only; it must never reach the bundle.""" + root = Path(__file__).resolve().parents[1] + for asset in list((root / "static").glob("*.js")) + [root / "templates" / "index.html"]: + text = asset.read_text() + assert "DEEPGRAM_API_KEY" not in text, f"key name leaked in {asset.name}" + assert "ELEVENLABS_API_KEY" not in text, f"key name leaked in {asset.name}" From 30ac655412c62d686b9ad343150966de7c11a84f Mon Sep 17 00:00:00 2001 From: Jake Lasky Date: Sat, 25 Jul 2026 01:07:15 -0400 Subject: [PATCH 05/10] chore: retire the v1 Flask remnants and fix the SDK-bump workflow The repo still carried its Deepgram-starter-template skin and a duplicate dependency pin from the pre-v2 Flask/gevent stack. Both were actively wrong rather than merely untidy. - .github/workflows/update-deepgram-sdk.yaml rewritten. It ran weekly, rewrote the pin in requirements.txt, then pip-installed that file (Flask 3.0.0, deepgram-sdk v3.10.1) on Python 3.10 for a project requiring >=3.12, then ran pytest. It could only no-op or open a broken PR, and it left roughly a dozen abandoned update-deepgram-sdk-v* branches on the remote. Now uses uv, reads the interpreter from requires-python, bumps pyproject + uv.lock, gates the PR on the suite passing, and uses create-pull-request so it stops littering branches. - requirements.txt and requirements-dev.txt deleted. pyproject.toml plus uv.lock is the single source of dependency truth, which is what the Dockerfile and README already used. Two homes for one pin is what the workflow above kept tripping over. - deepgram.toml rewritten. It still advertised the app as a Flask starter and told users to pip install requirements.txt. - pyproject renamed from flask-live-transcription to deepgram-stt-explorer, and [tool.uv] dev-dependencies migrated to PEP 735 [dependency-groups] since uv deprecated the old field. - Dropped the python-socketio[asyncio] extra. It does not exist, so uv warned on every command; .planning/STATE.md already records that async support is built in and not gated by an extra. - static/script.js deleted. Superseded by static/app.js and referenced nowhere. - The scripts/ evidence harness is now tracked, with its hardcoded /coding/... output paths replaced by a SCRIPT_DIR derived from __file__. Generated CSVs are gitignored so `git add -A` stops sweeping them in. - README documents the abuse limits and the deploy-drift hazard, and no longer claims 30 tests. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/update-deepgram-sdk.yaml | 139 +- .gitignore | 6 + README.md | 31 +- deepgram.toml | 15 +- pyproject.toml | 11 +- requirements-dev.txt | 5 - requirements.txt | 4 - sample.env | 1 + scripts/test_rrhaphy_keyterms.py | 142 ++ scripts/test_rrhaphy_multivoice.py | 247 +++ scripts/test_time_pronunciation.py | 307 +++ static/app.js | 2 +- static/script.js | 2134 -------------------- uv.lock | 36 +- 14 files changed, 824 insertions(+), 2256 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt create mode 100644 scripts/test_rrhaphy_keyterms.py create mode 100644 scripts/test_rrhaphy_multivoice.py create mode 100644 scripts/test_time_pronunciation.py delete mode 100644 static/script.js diff --git a/.github/workflows/update-deepgram-sdk.yaml b/.github/workflows/update-deepgram-sdk.yaml index edd5854..e5f6de7 100644 --- a/.github/workflows/update-deepgram-sdk.yaml +++ b/.github/workflows/update-deepgram-sdk.yaml @@ -1,105 +1,78 @@ name: Update Deepgram SDK +# Opens a PR when a newer deepgram-sdk is published. +# +# The pin lives in exactly ONE place: [project] dependencies in pyproject.toml, +# resolved through uv.lock. DO NOT reintroduce requirements.txt here. The +# previous version of this workflow edited requirements.txt and then ran +# `pip install -r requirements.txt`, which pulled the dead v1 Flask stack +# (Flask 3.0.0, deepgram-sdk v3.10.1) on Python 3.10 against a project that +# requires >=3.12. It could only no-op or open a broken PR, and it left roughly +# a dozen abandoned update-deepgram-sdk-v* branches on the remote. + on: schedule: - - cron: "0 0 * * 1" # Runs every Monday at midnight + - cron: "0 0 * * 1" # Every Monday at midnight UTC workflow_dispatch: +permissions: + contents: write + pull-requests: write + jobs: check-update: runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v5 with: - python-version: "3.10" - - - name: Install required tools - run: sudo apt-get install -y jq curl + enable-cache: true - - name: Get current Deepgram Python SDK version - id: get-deepgram-version - run: | - DEEPGRAM_VERSION=$(curl -s https://api.github.com/repos/deepgram/deepgram-python-sdk/releases/latest | jq -r '.tag_name') - echo "version=$DEEPGRAM_VERSION" >> $GITHUB_ENV + # Read the interpreter from pyproject's requires-python rather than + # hardcoding a version that can drift away from the project. + - name: Set up Python + run: uv python install - - name: Check installed Deepgram SDK version - id: check-installed-version + - name: Resolve current and latest deepgram-sdk + id: versions run: | - INSTALLED_VERSION=$(pip show deepgram-sdk | grep Version | cut -d' ' -f2) - echo "installed_version=$INSTALLED_VERSION" >> $GITHUB_ENV + set -euo pipefail + CURRENT=$(uv run python -c "import importlib.metadata as m; print(m.version('deepgram-sdk'))") + LATEST=$(curl -fsSL https://pypi.org/pypi/deepgram-sdk/json | jq -r '.info.version') + echo "current=$CURRENT" >> "$GITHUB_OUTPUT" + echo "latest=$LATEST" >> "$GITHUB_OUTPUT" + echo "Current: $CURRENT Latest: $LATEST" - - name: Config git - env: - GITHUB_TOKEN: ${{ secrets.GH_RELEASE_ACCESS_TOKEN }} - shell: bash + - name: Bump the pin and lockfile + if: steps.versions.outputs.current != steps.versions.outputs.latest run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global init.defaultBranch main - git config --global pull.rebase true - git config --global url."https://git:$GITHUB_TOKEN@github.com".insteadOf "https://github.com" + set -euo pipefail + uv add "deepgram-sdk==${{ steps.versions.outputs.latest }}" + uv lock - - name: Compare versions and update if necessary + - name: Run tests against the new version + if: steps.versions.outputs.current != steps.versions.outputs.latest env: - GITHUB_TOKEN: ${{ secrets.GH_RELEASE_ACCESS_TOKEN }} - run: | - LATEST_VERSION=${{ env.version }} - INSTALLED_VERSION=${{ env.installed_version }} + # A fake key is correct here: the suite asserts on wiring and accepts + # a Deepgram 401. It must never need a real credential to run. + DEEPGRAM_API_KEY: ci-test-key + run: uv run pytest tests/ -q - if [ "$LATEST_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Updating Deepgram SDK from $INSTALLED_VERSION to $LATEST_VERSION" - pip install deepgram-sdk==$LATEST_VERSION - - # Update requirements.txt - sed -i "s/deepgram-sdk==.*/deepgram-sdk==$LATEST_VERSION/" requirements.txt - git add requirements.txt - - # Create a new branch and commit changes - BRANCH_NAME="update-deepgram-sdk-$LATEST_VERSION" - git checkout -b "$BRANCH_NAME" - git commit -m "chore: update Deepgram SDK to $LATEST_VERSION" - - # Push the new branch and create a PR - git push origin "$BRANCH_NAME" - gh pr create --title "chore: update Deepgram SDK to $LATEST_VERSION" --body "This PR updates the Deepgram SDK to version $LATEST_VERSION." --base "main" --head "$BRANCH_NAME" - else - echo "Deepgram SDK is up to date" - fi - - - name: Install dependencies - run: pip install -r requirements.txt - - - name: Install dev dependencies - run: pip install -r requirements-dev.txt - - - name: Run tests - env: - DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} - run: pytest tests/ - - - name: Notify on failure - if: failure() - uses: slackapi/slack-github-action@v1.23.0 + - name: Open a PR + if: steps.versions.outputs.current != steps.versions.outputs.latest + uses: peter-evans/create-pull-request@v7 with: - payload: | - { - "text": "The tests have FAILED for ${{ github.repository }}." - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + branch: chore/deepgram-sdk-${{ steps.versions.outputs.latest }} + delete-branch: true + title: "chore: update deepgram-sdk to ${{ steps.versions.outputs.latest }}" + commit-message: "chore: update deepgram-sdk to ${{ steps.versions.outputs.latest }}" + body: | + Bumps `deepgram-sdk` from `${{ steps.versions.outputs.current }}` + to `${{ steps.versions.outputs.latest }}` in `pyproject.toml` and + `uv.lock`. - - name: Notify on success - if: success() - uses: slackapi/slack-github-action@v1.23.0 - with: - payload: | - { - "text": "The tests have passed for ${{ github.repository }}." - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + The test suite passed against the new version in CI. A major bump + can still change the streaming API shape, so check + `.planning/research/STACK.md` before merging one. diff --git a/.gitignore b/.gitignore index 99649c6..ee88d90 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ __pycache__/ .python-version .env outputs/* + +# Generated evidence from scripts/. Regenerable; keep deliberately, not by accident. +scripts/*.csv + +# Claude Code local overrides +.claude/settings.local.json diff --git a/README.md b/README.md index e54c989..cf84884 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,30 @@ Copy `sample.env` to `.env`: ```env DEEPGRAM_API_KEY=your_key_here +ELEVENLABS_API_KEY= # optional, only for the ElevenLabs TTS provider ``` +### Security and abuse limits + +This app holds a Deepgram key server-side and calls Deepgram on behalf of +whoever hits it, so these bound what a caller can do: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `ALLOWED_STT_HOSTS` | `api.deepgram.com` | Comma-separated allowlist of hosts the `base_url` override may target. A caller-supplied host outside this list is rejected with a 400. Add an AI Works host here to test a flow's `/v1/listen` compat endpoint. | +| `MAX_TTS_CHARS` | `2000` | Caps TTS text length. TTS bills per character, so this is a spend cap. | +| `MAX_UPLOAD_BYTES` | `104857600` | Caps upload size. | + +**`base_url` is operator-gated on purpose.** It used to accept any host and +forwarded the server's `Authorization: Token ` header there, which leaked +the key to a destination the caller picked. Do not relax the allowlist to +accept caller-supplied hosts. + +**Every endpoint is still unauthenticated.** The caps above limit the blast +radius; they do not stop an anonymous caller from spending the key's budget. +Put the app behind auth before sharing the URL widely, and use a scoped +Deepgram key rather than an org-wide one. + --- ## Supported Redact Values @@ -143,7 +165,7 @@ All values below are verified to work with the Deepgram streaming API: uv run pytest tests/ -v ``` -30 tests, 1 skipped. Tests use a real `UvicornTestServer` + `socketio.AsyncClient` — no mocking of the SocketIO layer. +58 tests, 1 skipped. Tests use a real `UvicornTestServer` + `socketio.AsyncClient` — no mocking of the SocketIO layer. @@ -161,6 +183,8 @@ Non-obvious things discovered during the Flask/gevent → FastAPI async migratio - **Audio `timeslice` must be 250ms** — 1000ms chunks cause the last word before Stop to be dropped - **`stream_started` must emit immediately on WS connect**, not after `Metadata` — Metadata timing is non-deterministic and can block the frontend for 10+ seconds - **Deepgram boolean params must be lowercase strings** (`"true"`/`"false"`), not Python bools +- **Never join a caller-supplied filename onto a directory** — Python's `Path` join *replaces* the base when the right side is absolute, so `TEMP_DIR / "/etc/passwd"` is `/etc/passwd`. Take `Path(name).name` and assert the resolved parent. +- **Never interpolate a caller-supplied host into a URL that carries your API key** — it forwards the credential to whatever host they named. --- @@ -172,6 +196,11 @@ fly secrets set DEEPGRAM_API_KEY=your_key_here fly deploy ``` +Pushing to `main` also deploys, via `.github/workflows/fly-deploy.yml`. That +workflow has no test gate, so run `uv run pytest tests/ -q` first. **Check +whether the running app is ahead of `main` before you push:** deploying `main` +while production carries newer code silently reverts it. + The `fly.toml` and `Dockerfile` are already configured for uvicorn on port 8080. --- diff --git a/deepgram.toml b/deepgram.toml index 327d6c8..5cbc23a 100644 --- a/deepgram.toml +++ b/deepgram.toml @@ -1,17 +1,20 @@ [meta] - title = "Flask Live Transcription Starter" - description = "Get started using Deepgram's Live Transcription with this Flask demo app" - author = "Deepgram DX Team (https://developers.deepgram.com)" + title = "Deepgram STT Explorer" + description = "Interactive app for exploring every Deepgram speech-to-text parameter over mic, file, batch, and TTS round-trip modes" + author = "Jacob-Lasky" useCase = "Live STT" language = "Python" - framework = "Flask" + framework = "FastAPI" [build] - command = "pip install -r requirements.txt" + # uv + pyproject.toml is the single source of dependency truth. + # DO NOT change this back to pip and requirements.txt: those files described + # the dead v1 Flask/gevent stack and were removed. + command = "uv sync --frozen" [config] sample = "sample.env" output = ".env" [post-build] - message = "Run `python app.py` to get up and running." # message to give users once setup is complete + message = "Run `uv run uvicorn app:app --port 8080` to get up and running." diff --git a/pyproject.toml b/pyproject.toml index 5bf1197..bcd75eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,14 @@ [project] -name = "flask-live-transcription" +name = "deepgram-stt-explorer" version = "0.1.0" -description = "" +description = "Interactive explorer for Deepgram speech-to-text parameters" authors = [{ name = "Jacob-Lasky", email = "jacob.s.lasky@gmail.com" }] requires-python = ">=3.12, <3.13" readme = "README.md" dependencies = [ "deepgram-sdk==6.0.1", "fastapi>=0.115.0,<1", - "python-socketio[asyncio]>=5.11.0,<6", + "python-socketio>=5.11.0,<6", "uvicorn>=0.30.0,<1", "httpx>=0.27.0,<1", "python-multipart>=0.0.9,<1", @@ -20,7 +20,10 @@ dependencies = [ [tool.uv] package = false -dev-dependencies = [ + +# uv deprecated [tool.uv] dev-dependencies in favour of PEP 735 dependency-groups. +[dependency-groups] +dev = [ "aiohttp>=3.9.0", "pytest>=8.0", "pytest-asyncio>=0.23,<1.0", diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f6c2f8d..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -# pip install -r requirements.txt - -# Testing -pytest -httpx \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6c5e685..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -deepgram-sdk==v3.10.1 -Flask==3.0.0 -Flask-SocketIO==5.3.6 -python-dotenv==1.0.0 diff --git a/sample.env b/sample.env index 3a84c6f..44881c7 100644 --- a/sample.env +++ b/sample.env @@ -1,2 +1,3 @@ DEEPGRAM_API_KEY=your_deepgram_api_key_here PORT=8001 +ELEVENLABS_API_KEY= diff --git a/scripts/test_rrhaphy_keyterms.py b/scripts/test_rrhaphy_keyterms.py new file mode 100644 index 0000000..1b791ae --- /dev/null +++ b/scripts/test_rrhaphy_keyterms.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +TTS→STT round-trip test for rare medical "-rrhaphy" terms. +Runs triplicates without keyterms, then triplicates with keyterms. +Uses the /api/tts-transcribe endpoint on deepgram-python-stt.fly.dev. +""" +import httpx +import json +import csv +import sys +from datetime import datetime +from pathlib import Path + +# Output lands next to this script. DO NOT hardcode an absolute /coding path: +# it pins the harness to one machine. +SCRIPT_DIR = Path(__file__).resolve().parent + +BASE_URL = "https://deepgram-python-stt.fly.dev" +TTS_MODEL = "aura-2-asteria-en" +STT_MODEL = "nova-3-medical" + +TERMS = [ + "perineorrhaphy", + "tarsorrhaphy", + "colporrhaphy", + "herniorrhaphy", + "capsulorrhaphy", + "splenorrhaphy", + "hymenorrhaphy", + "duodenorrhaphy", +] + +# Three different sentence templates per term for triplicates +TEMPLATES = [ + "The surgeon performed a {term} to repair the damaged tissue.", + "Following the examination, the physician recommended a {term} as the most appropriate intervention.", + "The operative report documented a successful {term} with no complications noted.", +] + + +def extract_transcript(result: dict) -> str: + """Pull transcript text from Deepgram batch response.""" + try: + return ( + result["results"]["channels"][0]["alternatives"][0]["transcript"] + ) + except (KeyError, IndexError): + return f"[ERROR: unexpected response shape: {json.dumps(result)[:200]}]" + + +def run_test(use_keyterms: bool) -> list[dict]: + """Run all term/template combos. Returns list of result dicts.""" + rows = [] + stt_params = {"model": STT_MODEL, "smart_format": True} + if use_keyterms: + # keyterms with intensifier boost + stt_params["keyterms"] = list(TERMS) + + label = "WITH keyterms" if use_keyterms else "WITHOUT keyterms" + print(f"\n{'='*60}") + print(f" Running {label}") + print(f"{'='*60}") + + with httpx.Client(timeout=120.0) as client: + for term in TERMS: + for i, template in enumerate(TEMPLATES): + sentence = template.format(term=term) + payload = { + "text": sentence, + "tts_model": TTS_MODEL, + "tts_provider": "deepgram", + "stt_params": stt_params, + "mode": "batch", + } + print(f"\n [{term}] trial {i+1}: {sentence}") + try: + resp = client.post(f"{BASE_URL}/api/tts-transcribe", json=payload) + resp.raise_for_status() + data = resp.json() + transcript = extract_transcript(data) + except Exception as e: + transcript = f"[REQUEST ERROR: {e}]" + + match = term.lower() in transcript.lower() + print(f" → {transcript}") + print(f" {'✓ MATCH' if match else '✗ MISS'}") + + rows.append({ + "term": term, + "trial": i + 1, + "input_sentence": sentence, + "transcript": transcript, + "match": match, + "keyterms": use_keyterms, + }) + return rows + + +def print_summary(rows: list[dict]): + """Print a summary table comparing with/without keyterms.""" + print(f"\n{'='*60}") + print(" SUMMARY") + print(f"{'='*60}") + print(f"\n{'Term':<22} {'No KT (3 trials)':<20} {'With KT (3 trials)':<20}") + print("-" * 62) + + for term in TERMS: + no_kt = sum(1 for r in rows if r["term"] == term and not r["keyterms"] and r["match"]) + with_kt = sum(1 for r in rows if r["term"] == term and r["keyterms"] and r["match"]) + print(f"{term:<22} {no_kt}/3{' ':>15} {with_kt}/3") + + no_kt_total = sum(1 for r in rows if not r["keyterms"] and r["match"]) + with_kt_total = sum(1 for r in rows if r["keyterms"] and r["match"]) + total_each = len(TERMS) * 3 + print("-" * 62) + print(f"{'TOTAL':<22} {no_kt_total}/{total_each}{' ':>14} {with_kt_total}/{total_each}") + + +def main(): + print(f"TTS→STT Round-Trip: -rrhaphy Medical Terms") + print(f"TTS model: {TTS_MODEL} | STT model: {STT_MODEL}") + print(f"Endpoint: {BASE_URL}/api/tts-transcribe") + print(f"Timestamp: {datetime.now().isoformat()}") + + all_rows = [] + all_rows.extend(run_test(use_keyterms=False)) + all_rows.extend(run_test(use_keyterms=True)) + + print_summary(all_rows) + + # Save CSV + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + csv_path = str(SCRIPT_DIR / f"rrhaphy_test_{ts}.csv") + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=all_rows[0].keys()) + writer.writeheader() + writer.writerows(all_rows) + print(f"\nResults saved to {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_rrhaphy_multivoice.py b/scripts/test_rrhaphy_multivoice.py new file mode 100644 index 0000000..71578fc --- /dev/null +++ b/scripts/test_rrhaphy_multivoice.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +TTS→STT round-trip test for rare medical "-rrhaphy" terms. +10 Deepgram TTS voices × 8 terms × 3 trials × 2 conditions (no keyterms / with keyterms). +Uses /api/tts-transcribe on deepgram-python-stt.fly.dev. +""" +import httpx +import csv +import sys +from datetime import datetime +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed + +# Output lands next to this script. DO NOT hardcode an absolute /coding path: +# it pins the harness to one machine. +SCRIPT_DIR = Path(__file__).resolve().parent + +BASE_URL = "https://deepgram-python-stt.fly.dev" +STT_MODEL = "nova-3-medical" + +VOICES = [ + "aura-2-asteria-en", + "aura-2-apollo-en", + "aura-2-arcas-en", + "aura-2-luna-en", + "aura-2-orion-en", + "aura-2-athena-en", + "aura-2-zeus-en", + "aura-2-hera-en", + "aura-2-orpheus-en", + "aura-2-thalia-en", +] + +TERMS = [ + "perineorrhaphy", + "tarsorrhaphy", + "colporrhaphy", + "herniorrhaphy", + "capsulorrhaphy", + "splenorrhaphy", + "hymenorrhaphy", + "duodenorrhaphy", +] + +TEMPLATES = [ + "The surgeon performed a {term} to repair the damaged tissue.", + "Following the examination, the physician recommended a {term} as the most appropriate intervention.", + "The operative report documented a successful {term} with no complications noted.", +] + + +def extract_transcript(result: dict) -> str: + try: + return result["results"]["channels"][0]["alternatives"][0]["transcript"] + except (KeyError, IndexError): + return "[ERROR]" + + +def classify_match(term: str, transcript: str) -> str: + """Classify how the STT handled the term. + Returns: 'exact', 'single_r', or 'miss'. + """ + t_lower = transcript.lower() + term_lower = term.lower() + if term_lower in t_lower: + return "exact" + # Build single-r variant: e.g. "colporrhaphy" -> "colporaphy", "colporhaphy" + # The double-r can appear as "rrh" -> "rh" or "rr" -> "r" + single_r_variants = set() + # rrhaphy -> rhaphy + single_r_variants.add(term_lower.replace("rrhaphy", "rhaphy")) + # rrhaphy -> raphy + single_r_variants.add(term_lower.replace("rrhaphy", "raphy")) + # rrhaphy -> rraphy (dropped h) + single_r_variants.add(term_lower.replace("rrhaphy", "rraphy")) + # rrhaphy -> raphe (alternate ending) + single_r_variants.add(term_lower.replace("rrhaphy", "raphe")) + # orr -> or (single r in the join) + single_r_variants.add(term_lower.replace("orrh", "orh")) + single_r_variants.add(term_lower.replace("orrh", "or")) + # Also check without "aphy" ending variations + stem = term_lower.replace("rrhaphy", "") + single_r_variants.add(stem + "rophy") + single_r_variants.add(stem + "rphy") + single_r_variants.add(stem + "rafy") + single_r_variants.add(stem + "raphy") + single_r_variants.add(stem + "rrhafy") + single_r_variants.add(stem + "rafi") + single_r_variants.add(stem + "rafy") + single_r_variants.add(stem + "raphey") + single_r_variants.add(stem + "rafe") + single_r_variants.discard(term_lower) # don't count exact as single_r + for variant in single_r_variants: + if variant in t_lower: + return "single_r" + return "miss" + + +def run_single(client: httpx.Client, voice: str, term: str, trial: int, template: str, use_keyterms: bool) -> dict: + sentence = template.format(term=term) + stt_params = {"model": STT_MODEL, "smart_format": True} + if use_keyterms: + stt_params["keyterms"] = list(TERMS) + + payload = { + "text": sentence, + "tts_model": voice, + "tts_provider": "deepgram", + "stt_params": stt_params, + "mode": "batch", + } + try: + resp = client.post(f"{BASE_URL}/api/tts-transcribe", json=payload) + resp.raise_for_status() + transcript = extract_transcript(resp.json()) + except Exception as e: + transcript = f"[ERROR: {e}]" + + category = classify_match(term, transcript) + return { + "voice": voice, + "term": term, + "trial": trial, + "input_sentence": sentence, + "transcript": transcript, + "match": category, + "keyterms": use_keyterms, + } + + +def main(): + total = len(VOICES) * len(TERMS) * len(TEMPLATES) * 2 + print(f"TTS→STT Multi-Voice -rrhaphy Test") + print(f"STT: {STT_MODEL} | Voices: {len(VOICES)} | Terms: {len(TERMS)} | Trials: 3 | Conditions: 2") + print(f"Total requests: {total}") + print(f"Started: {datetime.now().isoformat()}\n") + + # Build all jobs + jobs = [] + for use_kt in [False, True]: + for voice in VOICES: + for term in TERMS: + for trial_idx, template in enumerate(TEMPLATES): + jobs.append((voice, term, trial_idx + 1, template, use_kt)) + + # Run in parallel — 20 workers keeps good throughput without hammering the server + all_rows = [None] * len(jobs) + done = 0 + + def _run_job(idx, job): + voice, term, trial, template, use_kt = job + with httpx.Client(timeout=120.0) as client: + return idx, run_single(client, voice, term, trial, template, use_kt) + + with ThreadPoolExecutor(max_workers=20) as pool: + futures = {pool.submit(_run_job, i, job): i for i, job in enumerate(jobs)} + for future in as_completed(futures): + idx, row = future.result() + all_rows[idx] = row + done += 1 + if done % 20 == 0 or done == total: + print(f" [{done}/{total}] completed...") + + def _counts(rows, kt_filter): + """Return (exact, single_r, miss) counts for a filtered set.""" + filtered = [r for r in rows if r["keyterms"] == kt_filter] + exact = sum(1 for r in filtered if r["match"] == "exact") + single_r = sum(1 for r in filtered if r["match"] == "single_r") + miss = sum(1 for r in filtered if r["match"] == "miss") + return exact, single_r, miss + + n_per_term = len(VOICES) * 3 + n_per_voice = len(TERMS) * 3 + n_total = len(TERMS) * len(VOICES) * 3 + + # Per-term summary + print(f"\n{'='*100}") + print(f" AGGREGATE SUMMARY (across {len(VOICES)} voices, 3 trials each = {n_per_term} samples/term)") + print(f"{'='*100}") + + hdr = (f"{'':>22}" + f" {'--- No Keyterms ---':^30}" + f" {'--- With Keyterms ---':^30}") + sub = (f"{'Term':<22}" + f" {'Exact':>6} {'Single-R':>9} {'Miss':>6}" + f" {'Exact':>6} {'Single-R':>9} {'Miss':>6}") + print(f"\n{hdr}") + print(sub) + print("-" * 90) + + totals = {"no_kt": [0, 0, 0], "kt": [0, 0, 0]} + for term in TERMS: + term_rows = [r for r in all_rows if r["term"] == term] + e0, s0, m0 = _counts(term_rows, False) + e1, s1, m1 = _counts(term_rows, True) + totals["no_kt"] = [totals["no_kt"][i] + v for i, v in enumerate([e0, s0, m0])] + totals["kt"] = [totals["kt"][i] + v for i, v in enumerate([e1, s1, m1])] + print(f"{term:<22}" + f" {e0:>3}/{n_per_term:<3} {s0:>6}/{n_per_term:<3} {m0:>3}/{n_per_term:<3}" + f" {e1:>3}/{n_per_term:<3} {s1:>6}/{n_per_term:<3} {m1:>3}/{n_per_term:<3}") + + print("-" * 90) + print(f"{'TOTAL':<22}" + f" {totals['no_kt'][0]:>3}/{n_total:<4}" + f"{totals['no_kt'][1]:>5}/{n_total:<4}" + f"{totals['no_kt'][2]:>3}/{n_total:<4}" + f" {totals['kt'][0]:>3}/{n_total:<4}" + f"{totals['kt'][1]:>5}/{n_total:<4}" + f"{totals['kt'][2]:>3}/{n_total:<4}") + + # Per-voice summary + print(f"\n{'='*100}") + print(f" PER-VOICE SUMMARY ({n_per_voice} samples per voice per condition)") + print(f"{'='*100}") + + hdr_v = (f"{'':>22}" + f" {'--- No Keyterms ---':^30}" + f" {'--- With Keyterms ---':^30}") + sub_v = (f"{'Voice':<22}" + f" {'Exact':>6} {'Single-R':>9} {'Miss':>6}" + f" {'Exact':>6} {'Single-R':>9} {'Miss':>6}") + print(f"\n{hdr_v}") + print(sub_v) + print("-" * 90) + + for voice in VOICES: + v_short = voice.replace("aura-2-", "").replace("-en", "") + voice_rows = [r for r in all_rows if r["voice"] == voice] + e0, s0, m0 = _counts(voice_rows, False) + e1, s1, m1 = _counts(voice_rows, True) + print(f"{v_short:<22}" + f" {e0:>3}/{n_per_voice:<3} {s0:>6}/{n_per_voice:<3} {m0:>3}/{n_per_voice:<3}" + f" {e1:>3}/{n_per_voice:<3} {s1:>6}/{n_per_voice:<3} {m1:>3}/{n_per_voice:<3}") + + # Save CSV + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + csv_path = str(SCRIPT_DIR / f"rrhaphy_multivoice_{ts}.csv") + with open(csv_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=all_rows[0].keys()) + writer.writeheader() + writer.writerows(all_rows) + print(f"\nResults saved to {csv_path}") + print(f"Finished: {datetime.now().isoformat()}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_time_pronunciation.py b/scripts/test_time_pronunciation.py new file mode 100644 index 0000000..474bd07 --- /dev/null +++ b/scripts/test_time_pronunciation.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +""" +TTS→STT round-trip test for time pronunciation. + +Tests whether Deepgram TTS mispronounces times (e.g., "three fifty-one AM" +spoken as "three hundred fifty-one AM"). Generates audio via TTS, transcribes +via STT, and flags any transcript containing "hundred". + +Usage: + uv run scripts/test_time_pronunciation.py + uv run scripts/test_time_pronunciation.py --hours 8 9 10 --minutes 0 15 30 45 + uv run scripts/test_time_pronunciation.py --all # every minute 8AM-6PM + uv run scripts/test_time_pronunciation.py --voice aura-2-thalia-en + uv run scripts/test_time_pronunciation.py --concurrency 20 +""" +import argparse +import asyncio +import csv +import io +import json +import os +import sys +import time as time_mod +from datetime import datetime +from pathlib import Path + +import httpx +from dotenv import load_dotenv + +load_dotenv() + +# --- Number-to-words conversion (simple, covers 0-59 and hours 1-12) --- + +# Output lands next to this script. DO NOT hardcode an absolute /coding path: +# it pins the harness to one machine. +SCRIPT_DIR = Path(__file__).resolve().parent + +ONES = [ + "", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", + "seventeen", "eighteen", "nineteen", +] +TENS = ["", "", "twenty", "thirty", "forty", "fifty"] + + +def num_to_words(n: int) -> str: + if n < 20: + return ONES[n] + ten, one = divmod(n, 10) + return f"{TENS[ten]}-{ONES[one]}" if one else TENS[ten] + + +def time_to_spelled(hour: int, minute: int, period: str) -> str: + """Convert time to spelled-out words. e.g. (3, 51, 'AM') -> 'three fifty-one AM'""" + h = num_to_words(hour) + if minute == 0: + return f"{h} {period}" + elif minute < 10: + return f"{h} oh {num_to_words(minute)} {period}" + else: + return f"{h} {num_to_words(minute)} {period}" + + +def time_to_numeral(hour: int, minute: int, period: str) -> str: + """e.g. (3, 51, 'AM') -> '3:51 AM'""" + return f"{hour}:{minute:02d} {period}" + + +def make_sentence(time_str: str) -> str: + return ( + f"I'm reaching out regarding your appointment tomorrow at {time_str}, " + f"will you be joining us at that time?" + ) + + +# --- API calls --- + +TTS_URL = "https://api.deepgram.com/v1/speak" +STT_URL = "https://api.deepgram.com/v1/listen" + + +async def tts_then_stt( + client: httpx.AsyncClient, + api_key: str, + text: str, + voice: str, +) -> str: + """Send text to TTS, pipe audio to STT, return transcript.""" + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + + # TTS + tts_resp = await client.post( + TTS_URL, + params={"model": voice, "encoding": "mp3"}, + headers=headers, + json={"text": text}, + timeout=30, + ) + tts_resp.raise_for_status() + audio = tts_resp.content + + # STT (pre-recorded) + stt_headers = { + "Authorization": f"Token {api_key}", + "Content-Type": "audio/mpeg", + } + stt_resp = await client.post( + STT_URL, + params={"model": "nova-3", "smart_format": "true", "punctuate": "true"}, + headers=stt_headers, + content=audio, + timeout=30, + ) + stt_resp.raise_for_status() + data = stt_resp.json() + + transcript = ( + data.get("results", {}) + .get("channels", [{}])[0] + .get("alternatives", [{}])[0] + .get("transcript", "") + ) + return transcript + + +# --- Test runner --- + + +async def run_test( + sem: asyncio.Semaphore, + client: httpx.AsyncClient, + api_key: str, + voice: str, + hour: int, + minute: int, + period: str, + fmt: str, + results: list, + progress: dict, +): + if fmt == "spelled": + time_str = time_to_spelled(hour, minute, period) + else: + time_str = time_to_numeral(hour, minute, period) + + sentence = make_sentence(time_str) + + async with sem: + try: + transcript = await tts_then_stt(client, api_key, sentence, voice) + except Exception as e: + transcript = f"ERROR: {e}" + + has_hundred = "hundred" in transcript.lower() + + result = { + "hour": hour, + "minute": minute, + "period": period, + "format": fmt, + "input_time": time_str, + "input_sentence": sentence, + "transcript": transcript, + "has_hundred": has_hundred, + } + results.append(result) + + progress["done"] += 1 + flag = " *** HUNDRED DETECTED ***" if has_hundred else "" + print( + f" [{progress['done']}/{progress['total']}] " + f"{fmt:8s} {time_str:25s} -> {transcript[:80]}{flag}" + ) + + +async def main(): + parser = argparse.ArgumentParser(description="TTS→STT time pronunciation test") + parser.add_argument( + "--hours", nargs="+", type=int, default=None, + help="Hours to test (default: 8-12, 1-5 representative set)", + ) + parser.add_argument( + "--minutes", nargs="+", type=int, default=None, + help="Minutes to test (default: representative set)", + ) + parser.add_argument( + "--all", action="store_true", + help="Test every minute from 8AM to 6PM", + ) + parser.add_argument( + "--voice", default="aura-2-asteria-en", + help="TTS voice model (default: aura-2-asteria-en)", + ) + parser.add_argument( + "--concurrency", type=int, default=10, + help="Max concurrent API calls (default: 10)", + ) + parser.add_argument( + "--output", default=None, + help="CSV output file (default: auto-named in scripts/)", + ) + parser.add_argument( + "--formats", nargs="+", default=["spelled", "numeral"], + choices=["spelled", "numeral"], + help="Which input formats to test (default: both)", + ) + args = parser.parse_args() + + api_key = os.getenv("DEEPGRAM_API_KEY") + if not api_key: + print("Error: DEEPGRAM_API_KEY not set") + sys.exit(1) + + # Build time list + if args.all: + # 8:00 AM through 5:59 PM = hours 8-17 + times = [] + for h24 in range(8, 18): + for m in range(60): + period = "AM" if h24 < 12 else "PM" + h12 = h24 if h24 <= 12 else h24 - 12 + if h12 == 0: + h12 = 12 + times.append((h12, m, period)) + else: + hours = args.hours or [8, 9, 10, 11, 12, 1, 2, 3, 4, 5] + minutes = args.minutes or [ + 0, 1, 5, 10, 13, 15, 20, 21, 30, 31, 33, 40, 45, 50, 51, 55, 59, + ] + # Map hours to AM/PM + times = [] + for h in hours: + period = "AM" if h >= 8 and h <= 11 else "PM" + for m in minutes: + times.append((h, m, period)) + + total_tests = len(times) * len(args.formats) + print(f"\n=== Time Pronunciation Test ===") + print(f"Voice: {args.voice}") + print(f"Formats: {', '.join(args.formats)}") + print(f"Times: {len(times)} unique times x {len(args.formats)} formats = {total_tests} tests") + print(f"Concurrency: {args.concurrency}") + print() + + sem = asyncio.Semaphore(args.concurrency) + results = [] + progress = {"done": 0, "total": total_tests} + + start = time_mod.monotonic() + + async with httpx.AsyncClient() as client: + tasks = [] + for hour, minute, period in times: + for fmt in args.formats: + tasks.append( + run_test(sem, client, api_key, args.voice, hour, minute, period, fmt, results, progress) + ) + await asyncio.gather(*tasks) + + elapsed = time_mod.monotonic() - start + + # Summary + hundred_cases = [r for r in results if r["has_hundred"]] + errors = [r for r in results if r["transcript"].startswith("ERROR")] + + print(f"\n{'='*60}") + print(f"RESULTS SUMMARY") + print(f"{'='*60}") + print(f"Total tests: {len(results)}") + print(f"Errors: {len(errors)}") + print(f"'Hundred' found: {len(hundred_cases)} / {len(results) - len(errors)} successful") + print(f"Time elapsed: {elapsed:.1f}s") + + if hundred_cases: + print(f"\n{'='*60}") + print(f"FAILURES (transcript contains 'hundred'):") + print(f"{'='*60}") + for r in hundred_cases: + print(f" [{r['format']:8s}] Input: {r['input_time']}") + print(f" STT: {r['transcript']}") + print() + + # Breakdown by format + for fmt in args.formats: + fmt_results = [r for r in results if r["format"] == fmt and not r["transcript"].startswith("ERROR")] + fmt_hundreds = [r for r in fmt_results if r["has_hundred"]] + pct = (len(fmt_hundreds) / len(fmt_results) * 100) if fmt_results else 0 + print(f" {fmt:8s}: {len(fmt_hundreds)}/{len(fmt_results)} contain 'hundred' ({pct:.1f}%)") + + # Save CSV + if not args.output: + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + args.output = str(SCRIPT_DIR / f"time_pronunciation_{ts}.csv") + + with open(args.output, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=[ + "hour", "minute", "period", "format", "input_time", + "input_sentence", "transcript", "has_hundred", + ]) + writer.writeheader() + writer.writerows(sorted(results, key=lambda r: (r["format"], r["hour"], r["minute"]))) + + print(f"\nResults saved to: {args.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/static/app.js b/static/app.js index c71eeb7..1ab6103 100644 --- a/static/app.js +++ b/static/app.js @@ -4,7 +4,7 @@ function appData() { return { // ---- State ---- - mode: 'mic', // 'mic' | 'file' | 'batch' + mode: 'mic', // 'mic' | 'file' | 'batch' | 'tts' rightTab: 'transcript', connected: false, socket: null, diff --git a/static/script.js b/static/script.js deleted file mode 100644 index c5a78f3..0000000 --- a/static/script.js +++ /dev/null @@ -1,2134 +0,0 @@ -let isRecording = false; -let isStreamingFile = false; -let socket; -let microphone; -let audioContext; -let processor; -// Track which parameters have been changed during this session -let changedParams = new Set(); -// Track if we're in a post-import state -let isImported = false; - -// Global variables for progress tracking -let streamingStartTime = 0; -let streamingTimer = null; -let estimatedDuration = 0; -let audioPlayer = null; -let isAudioMuted = false; - -// Track if request ID has been shown for this session -let requestIdShown = false; - -// Define parameter compatibility -const PARAMETER_COMPATIBILITY = { - // Streaming-only parameters - streaming_only: ['interim_results', 'vad_events', 'endpointing', 'utterance_end'], - // Batch-only parameters - batch_only: ['paragraphs'] -}; - -// Function to show parameter notification -function showParameterNotification(message, type = 'info') { - // Remove any existing notification - const existingNotification = document.querySelector('.parameter-notification'); - if (existingNotification) { - existingNotification.remove(); - } - - // Create notification element - const notification = document.createElement('div'); - notification.className = `parameter-notification ${type}`; - notification.innerHTML = ` - - ${message} - - `; - - // Insert at the top of the config panel - const configPanel = document.querySelector('.config-panel'); - if (configPanel) { - configPanel.insertBefore(notification, configPanel.firstChild); - - // Auto-remove after 5 seconds - setTimeout(() => { - if (notification.parentElement) { - notification.remove(); - } - }, 5000); - } -} - -// Function to disable incompatible parameters and notify user -function handleParameterCompatibility(isStreaming) { - const disabledParams = []; - const incompatibleParams = isStreaming ? PARAMETER_COMPATIBILITY.batch_only : PARAMETER_COMPATIBILITY.streaming_only; - - incompatibleParams.forEach(paramId => { - const element = document.getElementById(paramId); - if (element) { - // If parameter was enabled, disable it and track for notification - if (element.type === 'checkbox' && element.checked) { - element.checked = false; - disabledParams.push(paramId); - } else if (element.type !== 'checkbox' && element.value.trim() !== '') { - element.value = ''; - disabledParams.push(paramId); - } - - // Disable the element - element.disabled = true; - - // Update tooltip - const mode = isStreaming ? 'streaming' : 'batch'; - const oppositeMode = isStreaming ? 'batch' : 'streaming'; - element.title = `${paramId} parameter is only available for ${oppositeMode} transcription, not ${mode}`; - } - }); - - // Show notification if any parameters were disabled - if (disabledParams.length > 0) { - const mode = isStreaming ? 'streaming' : 'batch'; - const paramList = disabledParams.map(p => `'${p}'`).join(', '); - const message = `Disabled ${paramList} parameter${disabledParams.length > 1 ? 's' : ''} - not supported in ${mode} mode`; - showParameterNotification(message, 'warning'); - - // Update URL to reflect parameter changes - updateRequestUrl(getConfig()); - } -} - -// Function to enable compatible parameters -function enableCompatibleParameters(isStreaming) { - const compatibleParams = isStreaming ? PARAMETER_COMPATIBILITY.streaming_only : PARAMETER_COMPATIBILITY.batch_only; - - compatibleParams.forEach(paramId => { - const element = document.getElementById(paramId); - if (element) { - element.disabled = false; - - // Restore normal tooltip - switch(paramId) { - case 'paragraphs': - element.title = 'Enable automatic paragraph formatting'; - break; - case 'interim_results': - element.title = 'Enable continuous transcription updates'; - break; - case 'vad_events': - element.title = 'Enable voice activity detection events'; - break; - case 'endpointing': - element.title = 'Speech finalization timing in milliseconds'; - break; - case 'utterance_end': - element.title = 'Utterance end detection timing in milliseconds'; - break; - default: - element.title = ''; - } - } - }); -} - -// Function to enable all parameters (neutral state) -function enableAllParameters() { - const allParams = [...PARAMETER_COMPATIBILITY.streaming_only, ...PARAMETER_COMPATIBILITY.batch_only]; - - allParams.forEach(paramId => { - const element = document.getElementById(paramId); - if (element) { - element.disabled = false; - - // Restore normal tooltip - switch(paramId) { - case 'paragraphs': - element.title = 'Enable automatic paragraph formatting'; - break; - case 'interim_results': - element.title = 'Enable continuous transcription updates'; - break; - case 'vad_events': - element.title = 'Enable voice activity detection events'; - break; - case 'endpointing': - element.title = 'Speech finalization timing in milliseconds'; - break; - case 'utterance_end': - element.title = 'Utterance end detection timing in milliseconds'; - break; - default: - element.title = ''; - } - } - }); -} - -// Function to disable paragraphs checkbox during streaming -function disableParagraphsForStreaming() { - handleParameterCompatibility(true); -} - -// Function to enable paragraphs checkbox when not streaming -function enableParagraphsForPrerecorded() { - enableAllParameters(); // Enable all parameters in neutral state -} - -// Function to stop file streaming -function stopFileStreaming() { - if (isStreamingFile) { - console.log('Stopping file streaming...'); - socket.emit('stop_stream_file'); - isStreamingFile = false; - document.body.classList.remove('recording'); - - // Reset the record button - const recordButton = document.getElementById('record'); - recordButton.checked = false; - - // Hide progress bar and stop audio - hideStreamingProgress(); - stopAudioPlayback(); - - // Re-enable paragraphs checkbox when not streaming - enableParagraphsForPrerecorded(); - - // Reset interim_results to the checkbox state - const config = getConfig(); - updateRequestUrl(config); - - console.log('File streaming stopped'); - } -} - -// Function to show streaming progress -function showStreamingProgress(filename, duration = 0) { - const progressContainer = document.getElementById('streamingProgress'); - const statusElement = document.getElementById('streamingStatus'); - const detailsElement = document.getElementById('streamingDetails'); - - estimatedDuration = duration; - streamingStartTime = Date.now(); - - statusElement.textContent = 'Streaming Audio'; - detailsElement.textContent = `File: ${filename}`; - - progressContainer.style.display = 'block'; - - // Start timer - startStreamingTimer(); -} - -// Function to hide streaming progress -function hideStreamingProgress() { - const progressContainer = document.getElementById('streamingProgress'); - progressContainer.style.display = 'none'; - - // Clear timer - if (streamingTimer) { - clearInterval(streamingTimer); - streamingTimer = null; - } -} - -// Function to update streaming progress -function updateStreamingProgress(status, details = '') { - const statusElement = document.getElementById('streamingStatus'); - const detailsElement = document.getElementById('streamingDetails'); - - if (status) statusElement.textContent = status; - if (details) detailsElement.textContent = details; -} - -// Function to start streaming timer -function startStreamingTimer() { - if (streamingTimer) clearInterval(streamingTimer); - - streamingTimer = setInterval(() => { - const elapsed = (Date.now() - streamingStartTime) / 1000; - const minutes = Math.floor(elapsed / 60); - const seconds = Math.floor(elapsed % 60); - - const timeElement = document.getElementById('streamingTime'); - timeElement.textContent = `${minutes}:${seconds.toString().padStart(2, '0')}`; - - // Update progress bar if we have estimated duration - if (estimatedDuration > 0) { - const progress = Math.min((elapsed / estimatedDuration) * 100, 100); - const progressFill = document.getElementById('progressFill'); - progressFill.style.width = `${progress}%`; - } - }, 1000); -} - -// Function to set progress bar to specific percentage -function setStreamingProgress(percentage) { - const progressFill = document.getElementById('progressFill'); - progressFill.style.width = `${Math.min(percentage, 100)}%`; -} - -// Function to setup audio player for streaming -function setupAudioPlayer(audioBlob, filename) { - const audioPlayerContainer = document.getElementById('audioPlayerContainer'); - const streamingAudio = document.getElementById('streamingAudio'); - const toggleAudioBtn = document.getElementById('toggleAudioBtn'); - - // Create object URL for the audio blob - const audioUrl = URL.createObjectURL(audioBlob); - streamingAudio.src = audioUrl; - - // Store reference to audio player - audioPlayer = streamingAudio; - - // Show audio player - audioPlayerContainer.style.display = 'block'; - - // Setup toggle button - toggleAudioBtn.onclick = toggleAudioMute; - - // Set initial volume - streamingAudio.volume = isAudioMuted ? 0 : 0.7; - - console.log('Audio player setup complete for:', filename); -} - -// Function to start synchronized audio playback -function startAudioPlayback() { - if (audioPlayer) { - audioPlayer.currentTime = 0; - audioPlayer.play().catch(error => { - console.log('Audio autoplay prevented:', error); - // Show a message to user that they need to click play - updateStreamingProgress('Ready to Play', 'Click the play button to hear audio while streaming'); - }); - } -} - -// Function to stop audio playback -function stopAudioPlayback() { - if (audioPlayer) { - audioPlayer.pause(); - audioPlayer.currentTime = 0; - } - - // Hide audio player - const audioPlayerContainer = document.getElementById('audioPlayerContainer'); - audioPlayerContainer.style.display = 'none'; - - // Clean up object URL - if (audioPlayer && audioPlayer.src) { - URL.revokeObjectURL(audioPlayer.src); - } - - audioPlayer = null; -} - -// Function to toggle audio mute -function toggleAudioMute() { - const toggleBtn = document.getElementById('toggleAudioBtn'); - const icon = toggleBtn.querySelector('i'); - - isAudioMuted = !isAudioMuted; - - if (audioPlayer) { - audioPlayer.volume = isAudioMuted ? 0 : 0.7; - } - - // Update button appearance - if (isAudioMuted) { - icon.className = 'fas fa-volume-mute'; - toggleBtn.classList.add('muted'); - } else { - icon.className = 'fas fa-volume-up'; - toggleBtn.classList.remove('muted'); - } -} - -let DEFAULT_CONFIG = { - "baseUrl": "api.deepgram.com", - "model": "nova-3", - "language": "en", - "utterance_end_ms": "1000", - "endpointing": "10", - "smart_format": false, - "interim_results": true, - "no_delay": false, - "dictation": false, - "numerals": false, - "profanity_filter": false, - "punctuate": false, - "multichannel": false, - "mip_opt_out": false, - "encoding": "", - "channels": 1, - "sample_rate": 16000, - "callback": "", - "keywords": "", - "replace": "", - "search": "", - "tags": "", - "version": "", - "redact": [], - "vad_events": true, - "diarize": false, - "filler_words": false, - "paragraphs": false, - "utterances": false, - "detect_entities": false, - "extra": {} -}; - -const socket_port = 8001; -socket = io( - "http://" + window.location.hostname + ":" + socket_port.toString() -); - -// Add Socket.IO connection event logging -socket.on('connect', () => { - console.log('Socket.IO connected successfully'); - console.log('Socket ID:', socket.id); -}); - -socket.on('disconnect', (reason) => { - console.log('Socket.IO disconnected:', reason); -}); - -socket.on('connect_error', (error) => { - console.error('Socket.IO connection error:', error); -}); - -socket.on('reconnect', (attemptNumber) => { - console.log('Socket.IO reconnected after', attemptNumber, 'attempts'); -}); - -socket.on('reconnect_error', (error) => { - console.error('Socket.IO reconnection error:', error); -}); - -// Listen for streaming events -socket.on('stream_started', (data) => { - console.log('Stream started:', data); - const filename = data.file_path ? data.file_path.split('/').pop() : 'Unknown file'; - showStreamingProgress(filename, data.duration || 0); - updateStreamingProgress('Streaming Audio', `Processing: ${filename}`); - - // Start audio playback if audio player is set up - if (audioPlayer) { - startAudioPlayback(); - } -}); - -socket.on('stream_error', (error) => { - console.error('Stream error:', error); - updateStreamingProgress('Error', `Streaming failed: ${error.error || 'Unknown error'}`); - - // Hide progress after showing error - setTimeout(() => { - if (isStreamingFile) { - stopFileStreaming(); - } - }, 3000); -}); - -socket.on('stream_finished', (data) => { - console.log('Stream finished:', data); - updateStreamingProgress('Completed', 'File streaming finished successfully'); - setStreamingProgress(100); - - // Hide progress after a delay - setTimeout(() => { - if (isStreamingFile) { - stopFileStreaming(); - } - }, 2000); -}); - -// Listen for real-time streaming responses -socket.on('streaming_response', (data) => { - console.log('Received streaming response:', data); - - // Display in a dedicated streaming responses area - const responsesContainer = document.getElementById('streaming-responses'); - if (responsesContainer) { - const responseDiv = document.createElement('div'); - responseDiv.className = 'streaming-response-item'; - responseDiv.innerHTML = ` -
- #${data.response_count} - ${new Date(data.timestamp * 1000).toLocaleTimeString()} - ${data.error ? '⚠️ Error' : ''} -
- 
${JSON.stringify(data.response, null, 2)}
- `; - - responsesContainer.appendChild(responseDiv); - responsesContainer.scrollTop = responsesContainer.scrollHeight; - } -}); - -// Listen for streaming responses saved notification -socket.on('responses_saved', (data) => { - console.log('Streaming responses saved:', data); - - // Show notification to user - const notification = document.createElement('div'); - notification.className = 'save-notification success'; - notification.innerHTML = ` - - Saved ${data.total_responses} streaming responses to ${data.filename} - `; - document.body.appendChild(notification); - - // Auto-remove notification after 5 seconds - setTimeout(() => { - notification.remove(); - }, 5000); -}); - -// Listen for streaming responses save errors -socket.on('responses_save_error', (data) => { - console.log('Error saving streaming responses:', data); - - const notification = document.createElement('div'); - notification.className = 'save-notification error'; - notification.innerHTML = ` - - ${data.message} - `; - document.body.appendChild(notification); - - setTimeout(() => { - notification.remove(); - }, 5000); -}); - -// Listen for transcript events from file streaming -socket.on('transcript', (data) => { - console.log('Received transcript:', data); - - const interimCaptions = document.getElementById("captions"); - const finalCaptions = document.getElementById("finalCaptions"); - - // Create interim message div with formatting similar to microphone streaming - const interimDiv = document.createElement("div"); - - // Build type indicator based on multiple flags - const indicators = []; - if (data.is_final) { - indicators.push("Is Final"); - } - if (data.speech_final) { - indicators.push("Speech Final"); - } - - // If no special indicators, it's an interim result - if (indicators.length === 0) { - indicators.push("Interim Result"); - } - - const type = `[${indicators.join(", ")}]`; - interimDiv.textContent = `${type} ${data.transcript}`; - interimDiv.className = data.is_final ? "final" : "interim"; - - // Add to interim container - interimCaptions.appendChild(interimDiv); - interimDiv.scrollIntoView({ behavior: "smooth" }); - - // Update final container - if (data.is_final) { - // Remove any existing interim span - const existingInterim = finalCaptions.querySelector('.interim-final'); - if (existingInterim) { - existingInterim.remove(); - } - // For final results, append as a new span - const finalDiv = document.createElement("span"); - finalDiv.textContent = data.transcript + " "; - finalDiv.className = "final"; - finalCaptions.appendChild(finalDiv); - - // Add line break if speech_final is true - if (data.speech_final) { - const lineBreak = document.createElement("br"); - finalCaptions.appendChild(lineBreak); - } - - finalDiv.scrollIntoView({ behavior: "smooth" }); - } else if (!data.speech_final) { - // For interim results, update or create the interim span (but not if speech_final) - let interimSpan = finalCaptions.querySelector('.interim-final'); - if (!interimSpan) { - interimSpan = document.createElement("span"); - interimSpan.className = "interim-final"; - finalCaptions.appendChild(interimSpan); - } - interimSpan.textContent = data.transcript; - interimSpan.scrollIntoView({ behavior: "smooth" }); - } -}); - -// Fetch default configuration -fetch('../config/defaults.json') - .then(response => response.json()) - .then(config => { - DEFAULT_CONFIG = config; - // Initialize URL with current config - updateRequestUrl(getConfig()); - }) - .catch(error => { - console.error('Error loading default configuration:', error); - // Initialize URL with current config - updateRequestUrl(getConfig()); - }); - -function setDefaultValues() { - if (!DEFAULT_CONFIG) return; - - // Set text input defaults - ['baseUrl', 'model', 'language', 'utterance_end_ms', 'endpointing', 'encoding', - 'callback', 'keywords', 'replace', 'search', 'tags', 'version'].forEach(id => { - const element = document.getElementById(id); - if (element && DEFAULT_CONFIG[id]) { - element.value = DEFAULT_CONFIG[id]; - } - }); - - // Set numeric input defaults - ['channels', 'sample_rate'].forEach(id => { - const element = document.getElementById(id); - if (element && DEFAULT_CONFIG[id] !== undefined) { - element.value = DEFAULT_CONFIG[id]; - } - }); - - // Set checkbox defaults - ['smart_format', 'interim_results', 'no_delay', 'dictation', - 'numerals', 'profanity_filter', 'punctuate', 'multichannel', 'mip_opt_out', - 'vad_events', 'diarize', 'filler_words', 'paragraphs', 'utterances', 'detect_entities'].forEach(id => { - const element = document.getElementById(id); - if (element && DEFAULT_CONFIG[id] !== undefined) { - element.checked = DEFAULT_CONFIG[id]; - } - }); - - // Set redact multi-select default - const redactElement = document.getElementById('redact'); - if (redactElement && DEFAULT_CONFIG.redact) { - Array.from(redactElement.options).forEach(option => { - option.selected = DEFAULT_CONFIG.redact.includes(option.value); - }); - } - - // Set extra params default - document.getElementById('extraParams').value = JSON.stringify(DEFAULT_CONFIG.extra || {}, null, 2); -} - -function resetConfig() { - if (!DEFAULT_CONFIG) return; - // Clear changed parameters tracking and import state - changedParams.clear(); - isImported = false; - setDefaultValues(); - updateRequestUrl(getConfig()); -} - -function importConfig(input) { - if (!DEFAULT_CONFIG) return; - - // Reset all options to defaults first - setDefaultValues(); - - let config; - - try { - config = JSON.parse(input); - } catch (e) { - config = parseUrlParams(input); - } - - if (!config) { - throw new Error('Invalid configuration format. Please provide a valid JSON object or URL.'); - } - - // Set import state - isImported = true; - - // Clear all form fields first - ['baseUrl', 'model', 'language', 'utterance_end_ms', 'endpointing', 'encoding', - 'callback', 'keywords', 'replace', 'search', 'tags', 'version'].forEach(id => { - const element = document.getElementById(id); - if (element) { - element.value = ''; - } - }); - - ['smart_format', 'interim_results', 'no_delay', 'dictation', - 'numerals', 'profanity_filter', 'punctuate', 'multichannel', 'mip_opt_out', - 'vad_events', 'diarize', 'filler_words', 'paragraphs', 'utterances', 'detect_entities'].forEach(id => { - const element = document.getElementById(id); - if (element) { - element.checked = false; - } - }); - - // Clear redact multi-select - const redactElement = document.getElementById('redact'); - if (redactElement) { - Array.from(redactElement.options).forEach(option => { - option.selected = false; - }); - } - - // Only set values that are explicitly in the config - Object.entries(config).forEach(([key, value]) => { - const element = document.getElementById(key); - if (element) { - if (element.type === 'checkbox') { - element.checked = value === 'true' || value === true; - } else if (key === 'redact' && element.multiple) { - // Handle redact multi-select - const values = Array.isArray(value) ? value : [value]; - Array.from(element.options).forEach(option => { - option.selected = values.includes(option.value); - }); - } else { - element.value = value; - } - changedParams.add(key); - } else { - // If the key doesn't correspond to a form element, it's an extra param - const extraParams = document.getElementById('extraParams'); - const currentExtra = JSON.parse(extraParams.value || '{}'); - currentExtra[key] = value; - extraParams.value = JSON.stringify(currentExtra, null, 2); - changedParams.add('extraParams'); - } - }); - - // Set baseUrl if not in config - if (!config.baseUrl) { - document.getElementById('baseUrl').value = 'api.deepgram.com'; - } - - // Update the URL display - updateRequestUrl(); -} - -socket.on("transcription_update", (data) => { - const interimCaptions = document.getElementById("captions"); - const finalCaptions = document.getElementById("finalCaptions"); - - let timeString = ""; - if (data.timing) { - const start = data.timing.start.toFixed(2); - const end = data.timing.end.toFixed(2); - timeString = `${start}-${end}`; - } - - // Create interim message div - const interimDiv = document.createElement("div"); - let type; - let showTimestamp = true; - - // Build type indicator based on multiple flags - const indicators = []; - if (data.is_final) { - indicators.push("Is Final"); - } - if (data.speech_final) { - indicators.push("Speech Final"); - } - if (data.utterance_end_ms) { - indicators.push("Utterance End"); - showTimestamp = false; // Don't show timestamp for utterance end - } - - // If no special indicators, it's an interim result - if (indicators.length === 0) { - indicators.push("Interim Result"); - } - - type = `[${indicators.join(", ")}]`; - - interimDiv.textContent = showTimestamp ? - `${timeString} ${type} ${data.transcription}` : - `${type} ${data.transcription}`; - interimDiv.className = data.is_final ? "final" : "interim"; - - // Add to interim container - interimCaptions.appendChild(interimDiv); - interimDiv.scrollIntoView({ behavior: "smooth" }); - - // Update final container - if (data.is_final) { - // Remove any existing interim span - const existingInterim = finalCaptions.querySelector('.interim-final'); - if (existingInterim) { - existingInterim.remove(); - } - // For final results, append as a new span - const finalDiv = document.createElement("span"); - finalDiv.textContent = data.transcription + " "; - finalDiv.className = "final"; - finalCaptions.appendChild(finalDiv); - - // Add line break if speech_final is true - if (data.speech_final) { - const lineBreak = document.createElement("br"); - finalCaptions.appendChild(lineBreak); - } - - finalDiv.scrollIntoView({ behavior: "smooth" }); - } else if (!data.utterance_end_ms && !data.speech_final) { - // For interim results, update or create the interim span - let interimSpan = finalCaptions.querySelector('.interim-final'); - if (!interimSpan) { - interimSpan = document.createElement("span"); - interimSpan.className = "interim-final"; - finalCaptions.appendChild(interimSpan); - } - interimSpan.textContent = data.transcription + " "; - interimSpan.scrollIntoView({ behavior: "smooth" }); - } -}); - -// Listen for request ID updates -socket.on("request_id_update", (data) => { - console.log('Received request ID:', data.request_id); - - // Only show the request ID once per session - if (data.request_id && !requestIdShown) { - // Display the request ID in the interim results container - const interimCaptions = document.getElementById("captions"); - const requestIdDiv = document.createElement("div"); - requestIdDiv.className = "url-info"; - requestIdDiv.textContent = `Request ID: ${data.request_id}`; - interimCaptions.appendChild(requestIdDiv); - requestIdDiv.scrollIntoView({ behavior: "smooth" }); - - // Mark that we've shown the request ID for this session - requestIdShown = true; - } -}); - -// Listen for raw response data -// Create collapsible JSON tree -function createJsonTree(data, key = null, level = 0) { - const container = document.createElement('div'); - container.className = 'json-node'; - - if (data === null) { - container.innerHTML = `${key ? `${key}: ` : ''}null`; - return container; - } - - if (typeof data === 'string') { - container.innerHTML = `${key ? `${key}: ` : ''}"${escapeHtml(data)}"`; - return container; - } - - if (typeof data === 'number') { - container.innerHTML = `${key ? `${key}: ` : ''}${data}`; - return container; - } - - if (typeof data === 'boolean') { - container.innerHTML = `${key ? `${key}: ` : ''}${data}`; - return container; - } - - if (Array.isArray(data)) { - const header = document.createElement('div'); - header.className = 'json-header'; - header.innerHTML = ` - ▶ - ${key ? `${key}:` : ''} - [ - ${data.length} items - ] - `; - - const content = document.createElement('div'); - content.className = 'json-content collapsed'; - content.style.marginLeft = '20px'; - - data.forEach((item, index) => { - const itemNode = createJsonTree(item, index, level + 1); - content.appendChild(itemNode); - }); - - header.addEventListener('click', () => { - const toggle = header.querySelector('.json-toggle'); - const isCollapsed = content.classList.contains('collapsed'); - - if (isCollapsed) { - content.classList.remove('collapsed'); - toggle.textContent = '▼'; - } else { - content.classList.add('collapsed'); - toggle.textContent = '▶'; - } - }); - - container.appendChild(header); - container.appendChild(content); - return container; - } - - if (typeof data === 'object') { - const keys = Object.keys(data); - const header = document.createElement('div'); - header.className = 'json-header'; - header.innerHTML = ` - ▶ - ${key ? `${key}:` : ''} - { - ${keys.length} keys - } - `; - - const content = document.createElement('div'); - content.className = 'json-content collapsed'; - content.style.marginLeft = '20px'; - - keys.forEach(objKey => { - const itemNode = createJsonTree(data[objKey], objKey, level + 1); - content.appendChild(itemNode); - }); - - header.addEventListener('click', () => { - const toggle = header.querySelector('.json-toggle'); - const isCollapsed = content.classList.contains('collapsed'); - - if (isCollapsed) { - content.classList.remove('collapsed'); - toggle.textContent = '▼'; - } else { - content.classList.add('collapsed'); - toggle.textContent = '▶'; - } - }); - - container.appendChild(header); - container.appendChild(content); - return container; - } - - // Fallback for unknown types - container.innerHTML = `${key ? `${key}: ` : ''}${String(data)}`; - return container; -} - -// Helper function to escape HTML -function escapeHtml(text) { - const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; -} - -// Format transcript with speaker labels for diarization -function formatTranscriptWithSpeakers(data) { - // Check if this is a multichannel response - if (data.results && data.results.channels && data.results.channels.length > 1) { - // Handle multichannel audio - let formattedTranscript = ''; - data.results.channels.forEach((channel, channelIndex) => { - if (channel.alternatives && channel.alternatives[0]) { - const alt = channel.alternatives[0]; - formattedTranscript += `\n[Channel ${channelIndex + 1}]\n`; - - if (alt.words && alt.words.length > 0) { - // Group words by speaker within this channel - const speakerGroups = groupWordsBySpeaker(alt.words); - speakerGroups.forEach(group => { - formattedTranscript += `Speaker ${group.speaker}: ${group.text}\n`; - }); - } else { - formattedTranscript += alt.transcript + '\n'; - } - } - }); - return formattedTranscript.trim(); - } - - // Handle single channel with diarization - if (data.results && data.results.channels && data.results.channels[0]) { - const channel = data.results.channels[0]; - if (channel.alternatives && channel.alternatives[0]) { - const alt = channel.alternatives[0]; - - // Check if words have speaker information - if (alt.words && alt.words.length > 0 && alt.words[0].hasOwnProperty('speaker')) { - const speakerGroups = groupWordsBySpeaker(alt.words); - let formattedTranscript = ''; - speakerGroups.forEach(group => { - formattedTranscript += `Speaker ${group.speaker}: ${group.text}\n`; - }); - return formattedTranscript.trim(); - } else { - // No speaker information, return plain transcript - return alt.transcript; - } - } - } - - // Fallback to plain transcript - return data.results?.channels[0]?.alternatives[0]?.transcript || ''; -} - -// Group words by speaker for diarization -function groupWordsBySpeaker(words) { - const groups = []; - let currentGroup = null; - - words.forEach(word => { - const speaker = word.speaker !== undefined ? word.speaker : 0; - - if (!currentGroup || currentGroup.speaker !== speaker) { - // Start new speaker group - currentGroup = { - speaker: speaker, - words: [word], - text: word.punctuated_word || word.word - }; - groups.push(currentGroup); - } else { - // Add to current speaker group - currentGroup.words.push(word); - currentGroup.text += ' ' + (word.punctuated_word || word.word); - } - }); - - return groups; -} - -socket.on("raw_response", (data) => { - console.log('Received raw response:', data); - console.log('Request ID:', data.request_id); - console.log('Parameters:', data.parameters); - console.log('Parameters keys:', data.parameters ? Object.keys(data.parameters) : 'No parameters'); - - // Display the raw response in the interim results container - const interimCaptions = document.getElementById("captions"); - - // Create a collapsible raw response section - const rawResponseDiv = document.createElement("div"); - rawResponseDiv.className = "raw-response-container"; - - // Create header with toggle functionality - const headerDiv = document.createElement("div"); - headerDiv.className = "raw-response-header"; - - // Build header text with request_id if available - let headerText = `🔍 Raw Response (${data.type})`; - if (data.request_id) { - headerText += ` - Request ID: ${data.request_id}`; - } - - headerDiv.innerHTML = ` - ${headerText} - ▼ - `; - - // Create content area (initially collapsed) - const contentDiv = document.createElement("div"); - contentDiv.className = "raw-response-content collapsed"; - - // Add parameters section if available - console.log('Checking parameters section:', { - hasParameters: !!data.parameters, - parametersType: typeof data.parameters, - parametersKeys: data.parameters ? Object.keys(data.parameters) : 'N/A', - parametersLength: data.parameters ? Object.keys(data.parameters).length : 0 - }); - - if (data.parameters && Object.keys(data.parameters).length > 0) { - console.log('Creating parameters section...'); - const parametersDiv = document.createElement("div"); - parametersDiv.className = "raw-response-section"; - - const parametersHeader = document.createElement("h4"); - parametersHeader.textContent = "Parameters Used:"; - parametersHeader.style.color = "#47aca9"; - parametersHeader.style.marginBottom = "8px"; - - const parametersContainer = document.createElement("div"); - parametersContainer.className = "json-tree-container"; - - try { - const parametersTree = createJsonTree(data.parameters); - parametersContainer.appendChild(parametersTree); - } catch (e) { - const preElement = document.createElement("pre"); - preElement.className = "raw-response-data"; - preElement.textContent = JSON.stringify(data.parameters, null, 2); - parametersContainer.appendChild(preElement); - } - - parametersDiv.appendChild(parametersHeader); - parametersDiv.appendChild(parametersContainer); - contentDiv.appendChild(parametersDiv); - } - - // Add response data section - const responseDiv = document.createElement("div"); - responseDiv.className = "raw-response-section"; - - const responseHeader = document.createElement("h4"); - responseHeader.textContent = "Response Data:"; - responseHeader.style.color = "#47aca9"; - responseHeader.style.marginBottom = "8px"; - responseHeader.style.marginTop = "16px"; - - // Create collapsible JSON tree - const jsonContainer = document.createElement("div"); - jsonContainer.className = "json-tree-container"; - - try { - const jsonTree = createJsonTree(data.data); - jsonContainer.appendChild(jsonTree); - } catch (e) { - // Fallback to plain text if JSON parsing fails - const preElement = document.createElement("pre"); - preElement.className = "raw-response-data"; - preElement.textContent = String(data.data); - jsonContainer.appendChild(preElement); - } - - responseDiv.appendChild(responseHeader); - responseDiv.appendChild(jsonContainer); - contentDiv.appendChild(responseDiv); - - // Add toggle functionality - headerDiv.addEventListener('click', () => { - const isCollapsed = contentDiv.classList.contains('collapsed'); - if (isCollapsed) { - contentDiv.classList.remove('collapsed'); - headerDiv.querySelector('.raw-response-toggle').textContent = '▲'; - } else { - contentDiv.classList.add('collapsed'); - headerDiv.querySelector('.raw-response-toggle').textContent = '▼'; - } - }); - - rawResponseDiv.appendChild(headerDiv); - rawResponseDiv.appendChild(contentDiv); - - interimCaptions.appendChild(rawResponseDiv); - rawResponseDiv.scrollIntoView({ behavior: "smooth" }); -}); - -async function getMicrophone() { - try { - // Get the selected device ID from the dropdown if it exists - const microphoneSelect = document.getElementById('microphone-select'); - const constraints = { audio: true }; - - // If a specific microphone is selected, use that device ID - if (microphoneSelect && microphoneSelect.value) { - constraints.audio = { deviceId: { exact: microphoneSelect.value } }; - } - - const stream = await navigator.mediaDevices.getUserMedia(constraints); - return new MediaRecorder(stream, { mimeType: "audio/webm" }); - } catch (error) { - console.error("Error accessing microphone:", error); - throw error; - } -} - -async function openMicrophone(microphone, socket) { - return new Promise((resolve) => { - microphone.onstart = () => { - console.log("Client: Microphone opened"); - document.body.classList.add("recording"); - resolve(); - }; - microphone.ondataavailable = async (event) => { - console.log("client: microphone data received"); - if (event.data.size > 0) { - socket.emit("audio_stream", event.data); - } - }; - microphone.start(250); - }); -} - -async function startRecording() { - isRecording = true; - // Reset request ID flag for new session - requestIdShown = false; - // Disable paragraphs checkbox during streaming - disableParagraphsForStreaming(); - microphone = await getMicrophone(); - console.log("Client: Waiting to open microphone"); - - // Send configuration before starting microphone - const config = getConfig(); - // Force interim_results to true for live recording - config.interim_results = true; - - // Update the UI to show interim_results is true - document.getElementById('interim_results').checked = true; - - // Update the URL display to show interim_results=true - updateRequestUrl(config); - - console.log("DEBUG: Frontend config:", config); - console.log("DEBUG: Base URL being sent:", config.baseUrl); - - socket.emit("toggle_transcription", { action: "start", config: config, baseUrl: config.baseUrl }); - - // Display the URL in the interim results container - const interimCaptions = document.getElementById("captions"); - const urlDiv = document.createElement("div"); - urlDiv.className = "url-info"; - const url = document.getElementById('requestUrl').textContent - .replace(/\s+/g, '') // Remove all whitespace including newlines - .replace(/&/g, '&'); // Fix any HTML-encoded ampersands - urlDiv.textContent = `Using URL: ${url}`; - interimCaptions.appendChild(urlDiv); - urlDiv.scrollIntoView({ behavior: "smooth" }); - - await openMicrophone(microphone, socket); -} - -async function stopRecording() { - if (isRecording === true) { - microphone.stop(); - microphone.stream.getTracks().forEach((track) => track.stop()); // Stop all tracks - socket.emit("toggle_transcription", { action: "stop" }); - microphone = null; - isRecording = false; - console.log("Client: Microphone closed"); - document.body.classList.remove("recording"); - - // Re-enable paragraphs checkbox when not streaming - enableParagraphsForPrerecorded(); - - // Reset interim_results to the checkbox state - const config = getConfig(); - updateRequestUrl(config); - } -} - -function getConfig() { - const config = {}; - - const addIfSet = (id) => { - const element = document.getElementById(id); - const value = element.type === 'checkbox' ? element.checked : element.value; - if (value !== '' && value !== false) { - config[id] = value; - } - }; - - addIfSet('baseUrl'); - addIfSet('language'); - addIfSet('model'); - addIfSet('utterance_end_ms'); - addIfSet('endpointing'); - addIfSet('smart_format'); - addIfSet('interim_results'); - addIfSet('no_delay'); - addIfSet('dictation'); - addIfSet('numerals'); - addIfSet('profanity_filter'); - addIfSet('punctuate'); - addIfSet('multichannel'); - addIfSet('mip_opt_out'); - addIfSet('encoding'); - addIfSet('channels'); - addIfSet('sample_rate'); - addIfSet('callback'); - addIfSet('keywords'); - addIfSet('replace'); - addIfSet('search'); - addIfSet('tags'); - addIfSet('version'); - addIfSet('vad_events'); - addIfSet('diarize'); - addIfSet('filler_words'); - addIfSet('paragraphs'); - addIfSet('utterances'); - addIfSet('detect_entities'); - - // Handle redact multi-select specially - const redactElement = document.getElementById('redact'); - if (redactElement) { - const selectedValues = Array.from(redactElement.selectedOptions).map(option => option.value); - if (selectedValues.length > 0) { - config['redact'] = selectedValues; - } - } - - // Add extra parameters - const extraParams = document.getElementById('extraParams'); - if (extraParams && extraParams.value) { - try { - const extra = JSON.parse(extraParams.value); - Object.entries(extra).forEach(([key, value]) => { - if (value !== undefined && value !== '') { - config[key] = value; - } - }); - } catch (e) { - console.error('Error parsing extra parameters:', e); - } - } - - return config; -} - -function toggleConfig() { - const header = document.querySelector('.config-header'); - const content = document.getElementById('configContent'); - header.classList.toggle('collapsed'); - content.classList.toggle('collapsed'); -} - -function updateRequestUrl() { - const urlElement = document.getElementById('requestUrl'); - - const baseUrl = document.getElementById('baseUrl').value; - const params = new URLSearchParams(); - - // Only add parameters that are explicitly set - const language = document.getElementById('language').value; - if (language) params.append('language', language); - - const model = document.getElementById('model').value; - if (model) params.append('model', model); - - const utteranceEnd = document.getElementById('utterance_end_ms').value; - if (utteranceEnd) params.append('utterance_end_ms', utteranceEnd); - - const endpointing = document.getElementById('endpointing').value; - if (endpointing) params.append('endpointing', endpointing); - - const encoding = document.getElementById('encoding').value; - if (encoding) params.append('encoding', encoding); - - const channels = document.getElementById('channels').value; - if (channels) params.append('channels', channels); - - const sampleRate = document.getElementById('sample_rate').value; - if (sampleRate) params.append('sample_rate', sampleRate); - - const callback = document.getElementById('callback').value; - if (callback) params.append('callback', callback); - - const keywords = document.getElementById('keywords').value; - if (keywords) params.append('keywords', keywords); - - const replace = document.getElementById('replace').value; - if (replace) params.append('replace', replace); - - const search = document.getElementById('search').value; - if (search) params.append('search', search); - - const tags = document.getElementById('tags').value; - if (tags) params.append('tags', tags); - - const version = document.getElementById('version').value; - if (version) params.append('version', version); - - const smartFormat = document.getElementById('smart_format').checked; - if (smartFormat) params.append('smart_format', 'true'); - - const interimResults = document.getElementById('interim_results').checked; - if (interimResults) params.append('interim_results', 'true'); - - const noDelay = document.getElementById('no_delay').checked; - if (noDelay) params.append('no_delay', 'true'); - - const dictation = document.getElementById('dictation').checked; - if (dictation) params.append('dictation', 'true'); - - const numerals = document.getElementById('numerals').checked; - if (numerals) params.append('numerals', 'true'); - - const profanityFilter = document.getElementById('profanity_filter').checked; - if (profanityFilter) params.append('profanity_filter', 'true'); - - const redact = document.getElementById('redact').checked; - if (redact) params.append('redact', 'true'); - - const punctuate = document.getElementById('punctuate').checked; - if (punctuate) params.append('punctuate', 'true'); - - const multichannel = document.getElementById('multichannel').checked; - if (multichannel) params.append('multichannel', 'true'); - - const mipOptOut = document.getElementById('mip_opt_out').checked; - if (mipOptOut) params.append('mip_opt_out', 'true'); - - const vadEvents = document.getElementById('vad_events').checked; - if (vadEvents) params.append('vad_events', 'true'); - - // Handle redact multi-select - const redactElement = document.getElementById('redact'); - if (redactElement) { - const selectedValues = Array.from(redactElement.selectedOptions).map(option => option.value); - selectedValues.forEach(value => { - if (value) params.append('redact', value); - }); - } - - const diarize = document.getElementById('diarize').checked; - if (diarize) params.append('diarize', 'true'); - - const fillerWords = document.getElementById('filler_words').checked; - if (fillerWords) params.append('filler_words', 'true'); - - const paragraphs = document.getElementById('paragraphs').checked; - if (paragraphs) params.append('paragraphs', 'true'); - - const utterances = document.getElementById('utterances').checked; - if (utterances) params.append('utterances', 'true'); - - const detectEntities = document.getElementById('detect_entities').checked; - if (detectEntities) params.append('detect_entities', 'true'); - - // Add extra parameters if any - const extraParams = document.getElementById('extraParams'); - if (extraParams && extraParams.value) { - try { - const extra = JSON.parse(extraParams.value); - Object.entries(extra).forEach(([key, value]) => { - if (value !== undefined && value !== '') { - if (Array.isArray(value)) { - value.forEach(v => params.append(key, v)); - } else { - params.append(key, value); - } - } - }); - } catch (e) { - console.error('Invalid extra parameters JSON:', e); - } - } - - // Calculate maxLineLength for new parameters - const containerWidth = urlElement.parentElement.getBoundingClientRect().width; - const avgCharWidth = 8.5; - const safetyMargin = 40; - const maxLineLength = Math.floor((containerWidth - safetyMargin) / avgCharWidth); - - // Format URL with line breaks - const baseUrlDisplay = isRecording ? `ws://${baseUrl}/v1/listen?` : `http://${baseUrl}/v1/listen?`; - const pairs = params.toString().split('&'); - let currentLine = baseUrlDisplay; - const outputLines = []; - - pairs.forEach((pair, index) => { - const shouldBreakLine = currentLine !== baseUrlDisplay && - (currentLine.length + pair.length + 1 > maxLineLength); - - if (shouldBreakLine) { - outputLines.push(currentLine + '&'); - currentLine = pair; - } else { - currentLine += (currentLine === baseUrlDisplay ? '' : '&') + pair; - } - - if (index === pairs.length - 1) { - outputLines.push(currentLine); - } - }); - - urlElement.innerHTML = outputLines.join('\n'); - return outputLines.join('').replace(/&/g, '&'); -} - -function toggleExtraParams() { - const header = document.querySelector('.extra-params-header'); - const content = document.getElementById('extraParamsContent'); - header.classList.toggle('collapsed'); - content.classList.toggle('collapsed'); -} - -function parseUrlParams(url) { - try { - // Handle ws:// and wss:// protocols by temporarily replacing them - let modifiedUrl = url; - if (url.startsWith('ws://') || url.startsWith('wss://')) { - modifiedUrl = url.replace(/^ws:\/\//, 'http://').replace(/^wss:\/\//, 'https://'); - } - - // If URL starts with a path, prepend the default base URL - if (url.startsWith('/')) { - modifiedUrl = 'http://api.deepgram.com' + url; - } - - const urlObj = new URL(modifiedUrl); - const params = {}; - - // Extract the hostname as baseUrl, removing /v1/listen if present - params.baseUrl = urlObj.hostname; - - // Handle duplicate parameters as arrays - const paramMap = new Map(); - urlObj.searchParams.forEach((value, key) => { - const cleanKey = key.trim(); - const cleanValue = value.trim(); - if (cleanKey && cleanValue) { - if (paramMap.has(cleanKey)) { - const existingValue = paramMap.get(cleanKey); - paramMap.set(cleanKey, Array.isArray(existingValue) ? [...existingValue, cleanValue] : [existingValue, cleanValue]); - } else { - paramMap.set(cleanKey, cleanValue); - } - } - }); - - // Convert Map to object - paramMap.forEach((value, key) => { - params[key] = value; - }); - - return params; - } catch (e) { - console.error('Invalid URL:', e); - return null; - } -} - -function simplifyUrl() { - // Clear import state and changed params - isImported = false; - changedParams.clear(); - // Update URL to show only non-default params - updateRequestUrl(getConfig()); -} - -document.addEventListener("DOMContentLoaded", () => { - const recordButton = document.getElementById("record"); - const configPanel = document.querySelector('.config-panel'); - const copyButton = document.getElementById('copyUrl'); - const resetButton = document.getElementById('resetButton'); - const simplifyButton = document.getElementById('simplifyButton'); - const clearButton = document.getElementById('clearButton'); - - // Make URL editable - const urlElement = document.getElementById('requestUrl'); - urlElement.contentEditable = true; - urlElement.style.cursor = 'text'; - - // Clear button functionality - if (clearButton) { - clearButton.addEventListener('click', () => { - document.getElementById('captions').innerHTML = ''; - document.getElementById('finalCaptions').innerHTML = ''; - }); - } - - // Reset button functionality - if (resetButton) { - resetButton.addEventListener('click', resetConfig); - } - - // Simplify button functionality - if (simplifyButton) { - simplifyButton.addEventListener('click', simplifyUrl); - } - - // Copy URL functionality - copyButton.addEventListener('click', () => { - const url = document.getElementById('requestUrl').textContent - .replace(/\s+/g, '') // Remove all whitespace including newlines - .replace(/&/g, '&'); // Fix any HTML-encoded ampersands - navigator.clipboard.writeText(url).then(() => { - copyButton.classList.add('copied'); - setTimeout(() => copyButton.classList.remove('copied'), 1000); - }); - }); - - // Add event listeners to all config inputs with change tracking - const configInputs = document.querySelectorAll('#configForm input'); - configInputs.forEach(input => { - input.addEventListener('change', () => { - changedParams.add(input.id); - updateRequestUrl(getConfig()); - }); - if (input.type === 'text') { - input.addEventListener('input', () => { - changedParams.add(input.id); - updateRequestUrl(getConfig()); - }); - } - }); - - // Add event listener for extra params - document.getElementById('extraParams').addEventListener('blur', () => { - try { - const extraParams = document.getElementById('extraParams'); - const rawJson = extraParams.value || '{}'; - // Parse the raw JSON string to handle duplicate keys - const processedExtra = {}; ul - const lines = rawJson.split('\n'); - lines.forEach(line => { - const match = line.match(/"([^"]+)":\s*"([^"]+)"/); - if (match) { - const [, key, value] = match; - if (processedExtra[key]) { - if (Array.isArray(processedExtra[key])) { - processedExtra[key].push(value); - } else { - processedExtra[key] = [processedExtra[key], value]; - } - } else { - processedExtra[key] = value; - } - } - }); - // Update the textarea with the processed JSON - extraParams.value = JSON.stringify(processedExtra, null, 2); - // Mark extra params as changed if they're not empty - if (Object.keys(processedExtra).length > 0) { - changedParams.add('extraParams'); - } else { - changedParams.delete('extraParams'); - } - updateRequestUrl(); - } catch (e) { - console.warn('Invalid JSON in extra params'); - } - }); - - // Add resize listener to update URL formatting when window size changes - window.addEventListener('resize', () => { - updateRequestUrl(getConfig()); - }); - - // Initialize URL with current config instead of defaults - updateRequestUrl(getConfig()); - - // Function to populate the microphone dropdown with available devices - async function populateMicrophoneList() { - try { - // First get permission to access media devices - await navigator.mediaDevices.getUserMedia({ audio: true }); - - // Then enumerate devices - const devices = await navigator.mediaDevices.enumerateDevices(); - const microphoneSelect = document.getElementById('microphone-select'); - - // Store current selection if any - const currentSelection = microphoneSelect.value; - - // Clear all options except the default - while (microphoneSelect.options.length > 1) { - microphoneSelect.remove(1); - } - - // Add all audio input devices - const audioInputDevices = devices.filter(device => device.kind === 'audioinput'); - audioInputDevices.forEach(device => { - const option = document.createElement('option'); - option.value = device.deviceId; - option.text = device.label || `Microphone ${microphoneSelect.options.length}`; - microphoneSelect.appendChild(option); - - // If this was the previously selected device, select it again - if (device.deviceId === currentSelection) { - microphoneSelect.value = device.deviceId; - } - }); - - console.log(`Found ${audioInputDevices.length} audio input devices`); - } catch (error) { - console.error('Error accessing media devices:', error); - } - } - - // Populate the microphone list on page load - populateMicrophoneList(); - - // Add event listener for the refresh button - const refreshButton = document.getElementById('refreshMicrophoneList'); - if (refreshButton) { - refreshButton.addEventListener('click', populateMicrophoneList); - } - - recordButton.addEventListener("change", async () => { - if (recordButton.checked) { - // Only start microphone recording if not already streaming a file - if (!isStreamingFile) { - try { - await startRecording(); - } catch (error) { - console.error("Failed to start recording:", error); - recordButton.checked = false; - } - } - } else { - // Stop both microphone recording and file streaming - if (isRecording) { - await stopRecording(); - } - if (isStreamingFile) { - stopFileStreaming(); - } - } - }); - - // Initialize extra params as collapsed - const extraParamsHeader = document.querySelector('.extra-params-header'); - extraParamsHeader.classList.add('collapsed'); - - // Add import button handler - document.getElementById('importButton').addEventListener('click', () => { - const importInput = document.getElementById('importInput'); - const input = importInput.value.trim(); - if (!input) { - alert('Please enter a configuration to import.'); - return; - } - - try { - importConfig(input); - // Only clear input if import was successful - importInput.value = ''; - } catch (e) { - alert('Invalid configuration format. Please provide a valid JSON object or URL.'); - } - }); - - // Add keyboard shortcut (Enter key) for import input - document.getElementById('importInput').addEventListener('keypress', (e) => { - if (e.key === 'Enter') { - e.preventDefault(); - document.getElementById('importButton').click(); - } - }); - - // Add event listener for URL editing - document.getElementById('requestUrl').addEventListener('input', function(e) { - // Store cursor position - const selection = window.getSelection(); - const range = selection.getRangeAt(0); - const cursorOffset = range.startOffset; - - const url = this.textContent.replace(/\s+/g, '').replace(/&/g, '&'); - const config = parseUrlParams(url); - if (config) { - // Update form fields based on URL - Object.entries(config).forEach(([key, value]) => { - const element = document.getElementById(key); - if (element) { - if (element.type === 'checkbox') { - element.checked = value === 'true' || value === true; - } else { - element.value = value; - } - changedParams.add(key); - } - }); - - // Update extra parameters - const extraParams = {}; - Object.entries(config).forEach(([key, value]) => { - if (!document.getElementById(key)) { - extraParams[key] = value; - } - }); - document.getElementById('extraParams').value = JSON.stringify(extraParams, null, 2); - - // Update URL display with proper wrapping and escaping - updateRequestUrl(); - - // Restore cursor position - try { - const urlElement = document.getElementById('requestUrl'); - const newRange = document.createRange(); - newRange.setStart(urlElement.firstChild || urlElement, Math.min(cursorOffset, (urlElement.firstChild || urlElement).length)); - newRange.collapse(true); - selection.removeAllRanges(); - selection.addRange(newRange); - } catch (e) { - console.warn('Could not restore cursor position:', e); - } - } - }); - - // File upload handling - const uploadButton = document.getElementById('uploadButton'); - const audioFile = document.getElementById('audioFile'); - const dropZone = document.getElementById('dropZone'); - - // Debug: Log when elements are found - console.log('Upload button found:', !!uploadButton); - console.log('Audio file input found:', !!audioFile); - console.log('Drop zone found:', !!dropZone); - - uploadButton.addEventListener('click', () => { - console.log('Upload button clicked'); - audioFile.click(); - }); - - // Drag and drop handling - dropZone.addEventListener('dragover', (e) => { - e.preventDefault(); - dropZone.classList.add('dragover'); - }); - - dropZone.addEventListener('dragleave', () => { - dropZone.classList.remove('dragover'); - }); - - dropZone.addEventListener('drop', (e) => { - e.preventDefault(); - dropZone.classList.remove('dragover'); - - if (e.dataTransfer.files.length === 0) { - console.log('No files dropped'); - return; - } - - const file = e.dataTransfer.files[0]; - console.log(`Dropped file: ${file.name}, type: ${file.type}, size: ${file.size} bytes`); - - processFile(file); - }); - - // Click on drop zone to trigger file input - dropZone.addEventListener('click', () => { - audioFile.click(); - }); - - // File input change handler - audioFile.addEventListener('change', (e) => { - if (e.target.files.length === 0) { - console.log('No file selected'); - return; - } - - const file = e.target.files[0]; - console.log(`Selected file: ${file.name}, type: ${file.type}, size: ${file.size} bytes`); - - processFile(file); - }); - - // Function to process a file - function processFile(file) { - const streamCheckbox = document.getElementById('streamPrerecorded'); - const isStreamMode = streamCheckbox.checked; - - if (isStreamMode) { - streamPrerecordedFile(file); - } else { - uploadFile(file); - } - } - - // Function to upload file for batch processing - function uploadFile(file) { - // Handle parameter compatibility for batch processing - handleParameterCompatibility(false); // false = batch mode - - const reader = new FileReader(); - - reader.onload = function(e) { - console.log(`File loaded, data length: ${e.target.result.length}`); - const fileData = { - name: file.name, - data: e.target.result - }; - - // Get parameters from URL - const urlElement = document.getElementById('requestUrl'); - const urlText = urlElement.textContent; - const params = {}; - - // Parse URL parameters - const url = new URL(urlText.replace('ws://', 'http://')); - // Only include parameters that are explicitly in the URL - for (const [key, value] of url.searchParams) { - params[key] = value; - } - - console.log(`Sending file upload request with params:`, params); - - // Use regular HTTP POST instead of Socket.IO - fetch('/upload', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - file: fileData, - config: params, - baseUrl: getConfig().baseUrl - }) - }) - .then(response => response.json()) - .then(result => { - console.log(`Received response:`, result); - if (result.error) { - console.error('Upload error:', result.error); - return; - } - - // Display transcription with speaker formatting - const formattedTranscript = formatTranscriptWithSpeakers(result); - if (formattedTranscript) { - const finalCaptions = document.getElementById('finalCaptions'); - const finalDiv = document.createElement('div'); - - // Check if transcript has speaker labels (contains "Speaker ") - if (formattedTranscript.includes('Speaker ') || formattedTranscript.includes('[Channel ')) { - // Use pre-formatted text to preserve line breaks - finalDiv.style.whiteSpace = 'pre-line'; - finalDiv.textContent = formattedTranscript; - } else { - // Plain transcript, use span for inline display - finalDiv.textContent = formattedTranscript + ' '; - } - - finalDiv.className = 'final'; - finalCaptions.appendChild(finalDiv); - finalDiv.scrollIntoView({ behavior: 'smooth' }); - } - }) - .catch(error => { - console.error('Upload error:', error); - }); - }; - - reader.onerror = function(e) { - console.error('Error reading file:', e); - }; - - reader.onprogress = function(e) { - if (e.lengthComputable) { - console.log(`Reading file: ${Math.round((e.loaded / e.total) * 100)}%`); - } - }; - - console.log('Starting to read file...'); - reader.readAsDataURL(file); - } - - // Function to stream a prerecorded file - function streamPrerecordedFile(file) { - console.log(`Starting file streaming for: ${file.name}`); - - // Reset request ID flag for new session - requestIdShown = false; - // Disable paragraphs checkbox during streaming - disableParagraphsForStreaming(); - - // Get current configuration - const config = getConfig(); - // Force interim_results to true for file streaming - config.interim_results = true; - - // Update the UI to show interim_results is true - document.getElementById('interim_results').checked = true; - - // Update the URL display to show interim_results=true - updateRequestUrl(config); - - // Set streaming state and update UI - isStreamingFile = true; - const recordButton = document.getElementById('record'); - recordButton.checked = true; - document.body.classList.add('recording'); - - // Display the URL in the interim results container - const interimCaptions = document.getElementById('captions'); - const urlDiv = document.createElement('div'); - urlDiv.className = 'url-info'; - const url = document.getElementById('requestUrl').textContent - .replace(/\s+/g, '') // Remove all whitespace including newlines - .replace(/&/g, '&'); // Fix any HTML-encoded ampersands - urlDiv.textContent = `Streaming file: ${file.name} | Using URL: ${url}`; - interimCaptions.appendChild(urlDiv); - urlDiv.scrollIntoView({ behavior: 'smooth' }); - - console.log(`File streaming config:`, config); - - // Step 1: Upload file via HTTP (to avoid Socket.IO size limits) - const reader = new FileReader(); - reader.onload = function(e) { - console.log(`File loaded for upload, data length: ${e.target.result.length}`); - const fileData = { - name: file.name, - data: e.target.result - }; - - console.log('Uploading file for streaming...'); - - // Show initial progress - showStreamingProgress(file.name); - updateStreamingProgress('Uploading', `Preparing ${file.name} for streaming...`); - - // Setup audio player with the file for synchronized playback - const audioBlob = new Blob([new Uint8Array(atob(e.target.result.split(',')[1]).split('').map(c => c.charCodeAt(0)))], {type: file.type}); - setupAudioPlayer(audioBlob, file.name); - - // Upload file via HTTP POST - fetch('/upload_for_streaming', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - file: fileData, - config: config - }) - }) - .then(response => response.json()) - .then(result => { - console.log('File upload response:', result); - - if (result.error) { - console.error('File upload error:', result.error); - stopFileStreaming(); - return; - } - - // Step 2: Start streaming via Socket.IO with just the file path - console.log('Starting streaming session...'); - updateStreamingProgress('Connecting', 'Establishing connection to Deepgram...'); - - socket.emit('start_file_streaming', { - file_path: result.file_path, - config: config, - baseUrl: config.baseUrl - }, (response) => { - console.log('Start streaming response:', response); - - if (response && response.error) { - console.error('Start streaming error:', response.error); - stopFileStreaming(); - } - }); - }) - .catch(error => { - console.error('File upload error:', error); - stopFileStreaming(); - }); - }; - - reader.onerror = function(e) { - console.error('Error reading file for streaming:', e); - stopFileStreaming(); - }; - - console.log('Starting to read file for upload...'); - reader.readAsDataURL(file); - } - - // Add event listener for interim_results checkbox - const interimResultsCheckbox = document.getElementById('interim_results'); - if (interimResultsCheckbox) { - interimResultsCheckbox.addEventListener('change', function() { - // Only update URL if not recording or streaming - if (!isRecording && !isStreamingFile) { - updateRequestUrl(getConfig()); - } - }); - } - - // Handle detect audio settings button - const detectSettingsButton = document.getElementById('detectSettingsButton'); - const audioSettingsDisplay = document.getElementById('audioSettingsDisplay'); - const audioSettingsContent = document.getElementById('audioSettingsContent'); - - if (detectSettingsButton && audioSettingsDisplay && audioSettingsContent) { - // Listen for audio settings from server - socket.on('audio_settings', function(settings) { - console.log('Received audio settings:', settings); - - // Show the settings display - audioSettingsDisplay.style.display = 'block'; - - // Clear previous content - audioSettingsContent.innerHTML = ''; - - if (settings.error) { - audioSettingsContent.innerHTML = `
${settings.error}
`; - return; - } - - // Create HTML for settings - const settingsHTML = ` -
- Device: ${settings.device_name || 'Unknown'} -
-
- Sample Rate: ${settings.sample_rate ? settings.sample_rate.toFixed(0) + ' Hz' : 'Unknown'} -
-
- Encoding: ${settings.dtype || 'Unknown'} -
-
- Bit Depth: ${settings.bit_depth ? settings.bit_depth + ' bits' : 'Unknown'} -
-
- Bitrate: ${settings.bitrate ? (settings.bitrate / 1000).toFixed(1) + ' kbps' : 'Unknown'} -
-
- Channels: ${settings.max_input_channels || 'Unknown'} -
- `; - - audioSettingsContent.innerHTML = settingsHTML; - - // Auto-populate form fields if they exist - if (settings.sample_rate) { - const sampleRateInput = document.getElementById('sample_rate'); - if (sampleRateInput) { - sampleRateInput.value = Math.round(settings.sample_rate); - } - } - - if (settings.dtype) { - const encodingInput = document.getElementById('encoding'); - if (encodingInput) { - // Map numpy dtype to encoding format - let encoding = ''; - if (settings.dtype.includes('float')) { - encoding = 'LINEAR32F'; - } else if (settings.dtype.includes('int16')) { - encoding = 'LINEAR16'; - } else if (settings.dtype.includes('int32')) { - encoding = 'LINEAR32'; - } - encodingInput.value = encoding; - } - } - }); - - // Handle detect settings button click - detectSettingsButton.addEventListener('click', function() { - console.log('Detecting audio settings...'); - socket.emit('detect_audio_settings'); - - // Show loading state - audioSettingsDisplay.style.display = 'block'; - audioSettingsContent.innerHTML = '
Detecting audio settings...
'; - }); - } - - // Handle stream checkbox change - const streamCheckbox = document.getElementById('streamPrerecorded'); - - if (streamCheckbox && uploadButton && dropZone) { - streamCheckbox.addEventListener('change', function() { - if (this.checked) { - uploadButton.innerHTML = ' Stream Audio'; - dropZone.innerHTML = '

Drag & drop audio files to stream

'; - } else { - uploadButton.innerHTML = ' Upload Audio'; - dropZone.innerHTML = '

Drag & drop audio files here

'; - } - }); - } - - // Initialize all parameters as enabled (neutral state by default) - enableAllParameters(); -}); diff --git a/uv.lock b/uv.lock index 48e5eca..43a936d 100644 --- a/uv.lock +++ b/uv.lock @@ -178,23 +178,7 @@ wheels = [ ] [[package]] -name = "fastapi" -version = "0.135.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e7/7b/f8e0211e9380f7195ba3f3d40c292594fd81ba8ec4629e3854c353aaca45/fastapi-0.135.1.tar.gz", hash = "sha256:d04115b508d936d254cea545b7312ecaa58a7b3a0f84952535b4c9afae7668cd", size = 394962, upload-time = "2026-03-01T18:18:29.369Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/72/42e900510195b23a56bde950d26a51f8b723846bfcaa0286e90287f0422b/fastapi-0.135.1-py3-none-any.whl", hash = "sha256:46e2fc5745924b7c840f71ddd277382af29ce1cdb7d5eab5bf697e3fb9999c9e", size = 116999, upload-time = "2026-03-01T18:18:30.831Z" }, -] - -[[package]] -name = "flask-live-transcription" +name = "deepgram-stt-explorer" version = "0.1.0" source = { virtual = "." } dependencies = [ @@ -227,7 +211,7 @@ requires-dist = [ { name = "pydub", specifier = ">=0.25.1,<0.26" }, { name = "python-dotenv", specifier = "==1.0.0" }, { name = "python-multipart", specifier = ">=0.0.9,<1" }, - { name = "python-socketio", extras = ["asyncio"], specifier = ">=5.11.0,<6" }, + { name = "python-socketio", specifier = ">=5.11.0,<6" }, { name = "sounddevice", specifier = ">=0.5.2,<0.6" }, { name = "uvicorn", specifier = ">=0.30.0,<1" }, ] @@ -240,6 +224,22 @@ dev = [ { name = "websocket-client", specifier = ">=1.9.0" }, ] +[[package]] +name = "fastapi" +version = "0.135.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/7b/f8e0211e9380f7195ba3f3d40c292594fd81ba8ec4629e3854c353aaca45/fastapi-0.135.1.tar.gz", hash = "sha256:d04115b508d936d254cea545b7312ecaa58a7b3a0f84952535b4c9afae7668cd", size = 394962, upload-time = "2026-03-01T18:18:29.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/72/42e900510195b23a56bde950d26a51f8b723846bfcaa0286e90287f0422b/fastapi-0.135.1-py3-none-any.whl", hash = "sha256:46e2fc5745924b7c840f71ddd277382af29ce1cdb7d5eab5bf697e3fb9999c9e", size = 116999, upload-time = "2026-03-01T18:18:30.831Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" From 7f4dee2c8235b4f4a92d95dd0a844090cd1ccb1a Mon Sep 17 00:00:00 2001 From: Jake Lasky Date: Sat, 25 Jul 2026 01:39:42 -0400 Subject: [PATCH 06/10] feat(security): gate the API and the stream behind a shared access token The UI shell stays public on purpose: GET / and /static/* need no token, so a shared link is browsable and the params panel is explorable with no wall. Everything that SPENDS THE DEEPGRAM KEY now requires one: /upload, /files/*, /transcribe, /api/tts-transcribe, /api/tts-voices, and the SocketIO stream. The socket is gated deliberately, even though the page that opens it is public. Mic and file streaming spend the key over that socket, so leaving it open would be a free key-spend path straight through the open UI. There is no way to authenticate "our own page" as distinct from a user: the browser is attacker-controlled, so anyone who loads the page can read its network calls and replay them. - APP_ACCESS_TOKEN unset means no gate, which is what local dev and the test suite want. REQUIRE_AUTH=true (set in fly.toml) makes the app refuse to boot without a token, so a forgotten fly secret fails the deploy loudly instead of quietly serving an open API. - Token accepted as X-App-Token, Authorization: Bearer, or ?token=. The query form is not laziness: an