Add multilingual support: lang detection, update STT/TTS and reply ge…

app/audio_stt.py

@@ -5,28 +5,35 @@
 import tempfile
 from app.config import settings
 import openai
+from app.lang_detect import detect_language
 
 logger = logging.getLogger(__name__)
 openai.api_key = settings.OPENAI_API_KEY
 
-async def transcribe_audio_openai(audio_bytes: bytes, mime_type: str = "audio/ogg") -> str:
-    """Transcribe audio bytes using OpenAI Whisper (synchronous SDK called in thread).
-    Returns the transcript string.
+async def transcribe_audio_openai(audio_bytes: bytes, mime_type: str = "audio/ogg") -> tuple[str, str]:
+    """Transcribe audio bytes using OpenAI Whisper and detect language.
+    Returns (transcript, language_code).
     """
     loop = asyncio.get_event_loop()
     def _call():
         f = io.BytesIO(audio_bytes)
-        # Depending on SDK version: use openai.Audio.transcribe or openai.Whisper
         try:
             resp = openai.Audio.transcribe("whisper-1", f)
-            # newer SDKs return object with 'text'
             if isinstance(resp, dict):
-                return resp.get('text', '')
-            return getattr(resp, 'text', '')
+                text = resp.get('text', '')
+            else:
+                text = getattr(resp, 'text', '')
+            return text
         except Exception:
-            # fallback: try ChatCompletion with base64? Not implemented
+            # re-raise to be handled in async wrapper
             raise
-    return await loop.run_in_executor(None, _call)
+    try:
+        transcript = await loop.run_in_executor(None, _call)
+        lang = detect_language(transcript)
+        return transcript, lang
+    except Exception:
+        logger.exception("STT (Whisper) call failed")
+        return "", "en"
 
 
 def _convert_to_ogg_opus(input_bytes: bytes) -> bytes:

app/audio_tts.py

@@ -4,6 +4,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def synthesize_eleven(text: str, voice: str = "alloy") -> tuple[bytes, str]:
     """Synthesize text to speech using ElevenLabs (blocking HTTP call). Returns (bytes, mime_type).
     """
@@ -16,3 +17,51 @@ def synthesize_eleven(text: str, voice: str = "alloy") -> tuple[bytes, str]:
     r = requests.post(url, json=data, headers=headers, stream=True, timeout=30)
     r.raise_for_status()
     return r.content, r.headers.get("Content-Type", "audio/mpeg")
+
+
+def synthesize_google_tts(text: str, lang: str = "en-US") -> tuple[bytes, str]:
+    """Synthesize using Google Cloud Text-to-Speech if available.
+    Returns (audio_bytes, mime_type) or raises if the library is not installed.
+    """
+    try:
+        from google.cloud import texttospeech
+    except Exception as e:
+        raise RuntimeError("google-cloud-texttospeech not available") from e
+
+    client = texttospeech.TextToSpeechClient()
+    synthesis_input = texttospeech.SynthesisInput(text=text)
+    # choose a standard voice for the requested language
+    voice = texttospeech.VoiceSelectionParams(
+        language_code=lang,
+        ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL
+    )
+    audio_config = texttospeech.AudioConfig(audio_encoding=texttospeech.AudioEncoding.OGG_OPUS)
+    response = client.synthesize_speech(input=synthesis_input, voice=voice, audio_config=audio_config)
+    return response.audio_content, "audio/ogg"
+
+
+def synthesize_preferred(text: str, lang: str = "en") -> tuple[bytes, str]:
+    """Synthesize text to speech trying preferred providers based on language.
+    Strategy:
+      - Try ElevenLabs first (if configured)
+      - If it fails and GOOGLE_TTS_ENABLED is true, try Google Cloud TTS with the language code
+      - Otherwise raise and let the caller handle fallback
+    Returns (audio_bytes, mime_type)
+    """
+    # Try ElevenLabs
+    try:
+        audio, mime = synthesize_eleven(text, voice="alloy")
+        return audio, mime
+    except Exception:
+        logger.exception("ElevenLabs TTS failed or unavailable for language %s", lang)
+    # Fallback to Google if enabled
+    if getattr(settings, "GOOGLE_TTS_ENABLED", False):
+        try:
+            # Google expects a BCP-47 language tag like en-US. Try to map a 2-letter code to en-US style.
+            lang_tag = lang if '-' in lang else (lang + '-US') if lang == 'en' else lang + '-'+lang.upper()
+            audio, mime = synthesize_google_tts(text, lang_tag)
+            return audio, mime
+        except Exception:
+            logger.exception("Google TTS fallback failed for lang=%s", lang)
+    # If we reach here, no provider was available
+    raise RuntimeError("No TTS provider available or synthesis failed")

app/lang_detect.py

@@ -0,0 +1,20 @@
+from langdetect import detect, DetectorFactory
+import logging
+
+DetectorFactory.seed = 0
+logger = logging.getLogger(__name__)
+
+
+def detect_language(text: str) -> str:
+    """Detect language code (ISO 639-1) for a given text.
+
+    Returns language code (e.g., 'en', 'fr', 'sw') or 'en' on failure.
+    """
+    if not text or not isinstance(text, str):
+        return 'en'
+    try:
+        lang = detect(text)
+        return lang
+    except Exception:
+        logger.exception('Language detection failed, defaulting to en')
+        return 'en'

app/reply_generator.py

@@ -1,18 +1,10 @@
-import os
-import openai
 import asyncio
 import logging
 from typing import List
+import openai
 from app.config import settings
-from tenacity import retry, wait_exponential, stop_after_attempt, retry_if_exception_type
-
-try:
-    from aiolimiter import AsyncLimiter
-except Exception:
-    AsyncLimiter = None
 
 logger = logging.getLogger(__name__)
-
 openai.api_key = settings.OPENAI_API_KEY
 
 SYSTEM_PROMPT = (
@@ -21,23 +13,16 @@
     "Keep replies short (1-3 sentences) and actionable."
 )
 
-# Rate limiter (simple per-process limiter). If aiolimiter isn't installed, we'll fall back to no rate-limiting.
-_limiter = AsyncLimiter(max_rate=settings.LLM_RATE_LIMIT_RPS, time_period=1) if AsyncLimiter else None
 
-
-@retry(wait=wait_exponential(min=1, max=8), stop=stop_after_attempt(3), retry=retry_if_exception_type(Exception))
 def _call_openai_sync(messages, model="gpt-3.5-turbo"):
-    # Synchronous call to OpenAI ChatCompletion (run in thread) -- wrapped with retries
     resp = openai.ChatCompletion.create(model=model, messages=messages, max_tokens=settings.LLM_MAX_TOKENS, temperature=0.3)
-    # Compatibility: some SDKs return choices with message.content; adjust as needed
     try:
         return resp.choices[0].message.content.strip()
     except Exception:
-        # fallback for older SDK response shape
         return resp.choices[0].text.strip()
 
 
-async def generate_reply(incoming_text: str, history: List[dict]):
+async def generate_reply(incoming_text: str, history: List[dict], lang: str | None = None):
     # Short-circuit if no OpenAI key
     if not settings.OPENAI_API_KEY:
         logger.info("OPENAI_API_KEY not set; using fallback reply")
@@ -47,29 +32,27 @@ async def generate_reply(incoming_text: str, history: List[dict]):
     if history and len(history) > settings.LLM_MAX_HISTORY_MESSAGES:
         history = history[-settings.LLM_MAX_HISTORY_MESSAGES:]
 
-    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+    system = SYSTEM_PROMPT
+    if lang:
+        # ask LLM to reply in the detected language (give short instruction)
+        system = system + f" Reply in the user's language: {lang}."
+
+    messages = [{"role": "system", "content": system}]
     if history:
         for item in history:
             text = item.get("text", "")
             messages.append({"role": "user", "content": text})
 
     messages.append({"role": "user", "content": incoming_text})
 
-    # Acquire rate limiter token if available
+    # run blocking OpenAI call in a thread with timeout
+    loop = asyncio.get_event_loop()
     try:
-        if _limiter:
-            await _limiter.acquire()
-        # run blocking OpenAI call in a thread with timeout
-        loop = asyncio.get_event_loop()
-        try:
-            raw = await asyncio.wait_for(loop.run_in_executor(None, _call_openai_sync, messages, settings.LLM_MODEL), timeout=settings.LLM_TIMEOUT_SECONDS)
-            return raw
-        except asyncio.TimeoutError:
-            logger.exception("OpenAI call timed out")
-            return "Thanks — I received your message. Can you tell me more?"
+        raw = await asyncio.wait_for(loop.run_in_executor(None, _call_openai_sync, messages, settings.LLM_MODEL), timeout=settings.LLM_TIMEOUT_SECONDS)
+        return raw
+    except asyncio.TimeoutError:
+        logger.exception("OpenAI call timed out")
+        return "Thanks — I received your message. Can you tell me more?"
     except Exception:
         logger.exception("LLM generation failed")
         return "Thanks — I received your message. Can you tell me more?"
-    finally:
-        # No explicit release needed for aiolimiter
-        pass

docs/multilingual.md

@@ -0,0 +1,21 @@
+# Multilingual voice support notes
+
+This doc explains how multilingual support is implemented and how it is used in the voice pipeline.
+
+Key points
+
+- Language detection: `app.lang_detect.detect_language()` (langdetect) is used to detect language codes (ISO 639-1) from transcripts or text messages.
+- STT: OpenAI Whisper transcribes audio and we detect language from the transcript. The worker pipeline receives (transcript, lang).
+- LLM: generate_reply(...) takes an optional `lang` parameter and instructs the LLM to reply in that language.
+- TTS: synthesize_preferred(text, lang) tries ElevenLabs first and falls back to Google Cloud TTS (if enabled) for wider language coverage.
+- Conversation store: language codes are persisted with messages so the system can reuse the user's last-known language when detection is unreliable.
+
+Configuration
+
+- Install langdetect: `pip install langdetect`
+- Optional: enable Google TTS fallback by setting `GOOGLE_TTS_ENABLED=true` and providing Google credentials (google-cloud-texttospeech package and GOOGLE_APPLICATION_CREDENTIALS env var).
+
+Best practices
+
+- For short or single-word audio, language detection may be unreliable. The worker will fall back to the user's last detected language (if available) or 'en'.
+- Monitor `stt_language_detected_total` metrics to see which languages are common and tune TTS voices accordingly.