diff --git a/app/audio_stt.py b/app/audio_stt.py index bbe7d46..31f883e 100644 --- a/app/audio_stt.py +++ b/app/audio_stt.py @@ -5,28 +5,35 @@ import tempfile from app.config import settings import openai +from app.lang_detect import detect_language logger = logging.getLogger(__name__) openai.api_key = settings.OPENAI_API_KEY -async def transcribe_audio_openai(audio_bytes: bytes, mime_type: str = "audio/ogg") -> str: - """Transcribe audio bytes using OpenAI Whisper (synchronous SDK called in thread). - Returns the transcript string. +async def transcribe_audio_openai(audio_bytes: bytes, mime_type: str = "audio/ogg") -> tuple[str, str]: + """Transcribe audio bytes using OpenAI Whisper and detect language. + Returns (transcript, language_code). """ loop = asyncio.get_event_loop() def _call(): f = io.BytesIO(audio_bytes) - # Depending on SDK version: use openai.Audio.transcribe or openai.Whisper try: resp = openai.Audio.transcribe("whisper-1", f) - # newer SDKs return object with 'text' if isinstance(resp, dict): - return resp.get('text', '') - return getattr(resp, 'text', '') + text = resp.get('text', '') + else: + text = getattr(resp, 'text', '') + return text except Exception: - # fallback: try ChatCompletion with base64? Not implemented + # re-raise to be handled in async wrapper raise - return await loop.run_in_executor(None, _call) + try: + transcript = await loop.run_in_executor(None, _call) + lang = detect_language(transcript) + return transcript, lang + except Exception: + logger.exception("STT (Whisper) call failed") + return "", "en" def _convert_to_ogg_opus(input_bytes: bytes) -> bytes: diff --git a/app/audio_tts.py b/app/audio_tts.py index 0cf78ab..b69515b 100644 --- a/app/audio_tts.py +++ b/app/audio_tts.py @@ -4,6 +4,7 @@ logger = logging.getLogger(__name__) + def synthesize_eleven(text: str, voice: str = "alloy") -> tuple[bytes, str]: """Synthesize text to speech using ElevenLabs (blocking HTTP call). Returns (bytes, mime_type). """ @@ -16,3 +17,51 @@ def synthesize_eleven(text: str, voice: str = "alloy") -> tuple[bytes, str]: r = requests.post(url, json=data, headers=headers, stream=True, timeout=30) r.raise_for_status() return r.content, r.headers.get("Content-Type", "audio/mpeg") + + +def synthesize_google_tts(text: str, lang: str = "en-US") -> tuple[bytes, str]: + """Synthesize using Google Cloud Text-to-Speech if available. + Returns (audio_bytes, mime_type) or raises if the library is not installed. + """ + try: + from google.cloud import texttospeech + except Exception as e: + raise RuntimeError("google-cloud-texttospeech not available") from e + + client = texttospeech.TextToSpeechClient() + synthesis_input = texttospeech.SynthesisInput(text=text) + # choose a standard voice for the requested language + voice = texttospeech.VoiceSelectionParams( + language_code=lang, + ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL + ) + audio_config = texttospeech.AudioConfig(audio_encoding=texttospeech.AudioEncoding.OGG_OPUS) + response = client.synthesize_speech(input=synthesis_input, voice=voice, audio_config=audio_config) + return response.audio_content, "audio/ogg" + + +def synthesize_preferred(text: str, lang: str = "en") -> tuple[bytes, str]: + """Synthesize text to speech trying preferred providers based on language. + Strategy: + - Try ElevenLabs first (if configured) + - If it fails and GOOGLE_TTS_ENABLED is true, try Google Cloud TTS with the language code + - Otherwise raise and let the caller handle fallback + Returns (audio_bytes, mime_type) + """ + # Try ElevenLabs + try: + audio, mime = synthesize_eleven(text, voice="alloy") + return audio, mime + except Exception: + logger.exception("ElevenLabs TTS failed or unavailable for language %s", lang) + # Fallback to Google if enabled + if getattr(settings, "GOOGLE_TTS_ENABLED", False): + try: + # Google expects a BCP-47 language tag like en-US. Try to map a 2-letter code to en-US style. + lang_tag = lang if '-' in lang else (lang + '-US') if lang == 'en' else lang + '-'+lang.upper() + audio, mime = synthesize_google_tts(text, lang_tag) + return audio, mime + except Exception: + logger.exception("Google TTS fallback failed for lang=%s", lang) + # If we reach here, no provider was available + raise RuntimeError("No TTS provider available or synthesis failed") diff --git a/app/lang_detect.py b/app/lang_detect.py new file mode 100644 index 0000000..a1e87f4 --- /dev/null +++ b/app/lang_detect.py @@ -0,0 +1,20 @@ +from langdetect import detect, DetectorFactory +import logging + +DetectorFactory.seed = 0 +logger = logging.getLogger(__name__) + + +def detect_language(text: str) -> str: + """Detect language code (ISO 639-1) for a given text. + + Returns language code (e.g., 'en', 'fr', 'sw') or 'en' on failure. + """ + if not text or not isinstance(text, str): + return 'en' + try: + lang = detect(text) + return lang + except Exception: + logger.exception('Language detection failed, defaulting to en') + return 'en' diff --git a/app/reply_generator.py b/app/reply_generator.py index 328615c..b0bfba3 100644 --- a/app/reply_generator.py +++ b/app/reply_generator.py @@ -1,18 +1,10 @@ -import os -import openai import asyncio import logging from typing import List +import openai from app.config import settings -from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception_type - -try: - from aiolimiter import AsyncLimiter -except Exception: - AsyncLimiter = None logger = logging.getLogger(__name__) - openai.api_key = settings.OPENAI_API_KEY SYSTEM_PROMPT = ( @@ -21,23 +13,16 @@ "Keep replies short (1-3 sentences) and actionable." ) -# Rate limiter (simple per-process limiter). If aiolimiter isn't installed, we'll fall back to no rate-limiting. -_limiter = AsyncLimiter(max_rate=settings.LLM_RATE_LIMIT_RPS, time_period=1) if AsyncLimiter else None - -@retry(wait=wait_exponential(min=1, max=8), stop=stop_after_attempt(3), retry=retry_if_exception_type(Exception)) def _call_openai_sync(messages, model="gpt-3.5-turbo"): - # Synchronous call to OpenAI ChatCompletion (run in thread) -- wrapped with retries resp = openai.ChatCompletion.create(model=model, messages=messages, max_tokens=settings.LLM_MAX_TOKENS, temperature=0.3) - # Compatibility: some SDKs return choices with message.content; adjust as needed try: return resp.choices[0].message.content.strip() except Exception: - # fallback for older SDK response shape return resp.choices[0].text.strip() -async def generate_reply(incoming_text: str, history: List[dict]): +async def generate_reply(incoming_text: str, history: List[dict], lang: str | None = None): # Short-circuit if no OpenAI key if not settings.OPENAI_API_KEY: logger.info("OPENAI_API_KEY not set; using fallback reply") @@ -47,7 +32,12 @@ async def generate_reply(incoming_text: str, history: List[dict]): if history and len(history) > settings.LLM_MAX_HISTORY_MESSAGES: history = history[-settings.LLM_MAX_HISTORY_MESSAGES:] - messages = [{"role": "system", "content": SYSTEM_PROMPT}] + system = SYSTEM_PROMPT + if lang: + # ask LLM to reply in the detected language (give short instruction) + system = system + f" Reply in the user's language: {lang}." + + messages = [{"role": "system", "content": system}] if history: for item in history: text = item.get("text", "") @@ -55,21 +45,14 @@ async def generate_reply(incoming_text: str, history: List[dict]): messages.append({"role": "user", "content": incoming_text}) - # Acquire rate limiter token if available + # run blocking OpenAI call in a thread with timeout + loop = asyncio.get_event_loop() try: - if _limiter: - await _limiter.acquire() - # run blocking OpenAI call in a thread with timeout - loop = asyncio.get_event_loop() - try: - raw = await asyncio.wait_for(loop.run_in_executor(None, _call_openai_sync, messages, settings.LLM_MODEL), timeout=settings.LLM_TIMEOUT_SECONDS) - return raw - except asyncio.TimeoutError: - logger.exception("OpenAI call timed out") - return "Thanks — I received your message. Can you tell me more?" + raw = await asyncio.wait_for(loop.run_in_executor(None, _call_openai_sync, messages, settings.LLM_MODEL), timeout=settings.LLM_TIMEOUT_SECONDS) + return raw + except asyncio.TimeoutError: + logger.exception("OpenAI call timed out") + return "Thanks — I received your message. Can you tell me more?" except Exception: logger.exception("LLM generation failed") return "Thanks — I received your message. Can you tell me more?" - finally: - # No explicit release needed for aiolimiter - pass diff --git a/docs/multilingual.md b/docs/multilingual.md new file mode 100644 index 0000000..687b2aa --- /dev/null +++ b/docs/multilingual.md @@ -0,0 +1,21 @@ +# Multilingual voice support notes + +This doc explains how multilingual support is implemented and how it is used in the voice pipeline. + +Key points + +- Language detection: `app.lang_detect.detect_language()` (langdetect) is used to detect language codes (ISO 639-1) from transcripts or text messages. +- STT: OpenAI Whisper transcribes audio and we detect language from the transcript. The worker pipeline receives (transcript, lang). +- LLM: generate_reply(...) takes an optional `lang` parameter and instructs the LLM to reply in that language. +- TTS: synthesize_preferred(text, lang) tries ElevenLabs first and falls back to Google Cloud TTS (if enabled) for wider language coverage. +- Conversation store: language codes are persisted with messages so the system can reuse the user's last-known language when detection is unreliable. + +Configuration + +- Install langdetect: `pip install langdetect` +- Optional: enable Google TTS fallback by setting `GOOGLE_TTS_ENABLED=true` and providing Google credentials (google-cloud-texttospeech package and GOOGLE_APPLICATION_CREDENTIALS env var). + +Best practices + +- For short or single-word audio, language detection may be unreliable. The worker will fall back to the user's last detected language (if available) or 'en'. +- Monitor `stt_language_detected_total` metrics to see which languages are common and tune TTS voices accordingly.