feat(whatsapp): support inbound voice notes (#288)

src/config/env.ts

@@ -420,6 +420,25 @@ export const config = {
     twilioToken: process.env.TWILIO_AUTH_TOKEN || '',
     fromNumber: process.env.WHATSAPP_FROM || '',
   },
+  // Speech-to-text for WhatsApp voice notes (#288). The provider is swappable
+  // behind the TranscriptionProvider interface; these settings configure the
+  // default (OpenAI Whisper) implementation. Raw audio is never persisted —
+  // see docs/WHATSAPP_VOICE.md.
+  transcription: {
+    provider: process.env.TRANSCRIPTION_PROVIDER || 'openai',
+    openaiApiKey: process.env.OPENAI_API_KEY || '',
+    model: process.env.TRANSCRIPTION_MODEL || 'whisper-1',
+    apiUrl:
+      process.env.TRANSCRIPTION_API_URL ||
+      'https://api.openai.com/v1/audio/transcriptions',
+    /**
+     * Minimum transcription confidence (0–1) required to act on a voice note.
+     * Below this the bot asks the user to repeat or type rather than guessing.
+     */
+    confidenceThreshold: parseFloat(
+      process.env.TRANSCRIPTION_CONFIDENCE_THRESHOLD || '0.6'
+    ),
+  },
   dlq: {
     alertThreshold: parseInt(process.env.DLQ_ALERT_THRESHOLD || '50'),
     alertCooldownMs: parseInt(process.env.DLQ_ALERT_COOLDOWN_MS || '900000'), // 15 minutes default

src/routes/whatsapp.ts

@@ -65,8 +65,22 @@ router.post(
     const from = (req.body.From as string) || ''
     const body = (req.body.Body as string) || ''
 
+    // Voice notes (and other media) arrive with NumMedia > 0 and MediaUrl0 /
+    // MediaContentType0. We only act on audio; non-audio media falls through to
+    // text handling (empty body → 'unknown').
+    const numMedia = parseInt((req.body.NumMedia as string) || '0', 10)
+    const mediaUrl = req.body.MediaUrl0 as string | undefined
+    const mediaContentType = req.body.MediaContentType0 as string | undefined
+    const media =
+      numMedia > 0 &&
+      mediaUrl &&
+      mediaContentType &&
+      mediaContentType.startsWith('audio/')
+        ? { url: mediaUrl, contentType: mediaContentType }
+        : undefined
+
     try {
-      const response = await handleWhatsAppMessage(from, body)
+      const response = await handleWhatsAppMessage(from, body, media)
       const responseTwiml = new twiml.MessagingResponse()
       responseTwiml.message(response.body)
       res.type('text/xml').send(responseTwiml.toString())

src/validators/webhook-validators.ts

@@ -1,9 +1,30 @@
 import { z } from 'zod'
 
-export const whatsappWebhookSchema = z.object({
-  From: z.string().min(1, 'From is required'),
-  Body: z.string().min(1, 'Body is required'),
-})
+/**
+ * Twilio inbound WhatsApp webhook. `Body` is optional because a voice note
+ * arrives with media fields and an empty/absent body (#288). `NumMedia`/
+ * `MediaUrl0`/`MediaContentType0` are Twilio's media fields; we read the first
+ * attachment. `.passthrough()` keeps Twilio's other fields and — crucially — we
+ * add NO defaulted keys, so the object still matches the exact params Twilio
+ * signed (signature validation runs on req.body after this parse).
+ *
+ * A message must carry either text or media: `Body` alone being optional would
+ * let an empty payload through to the signature check, so the refinement below
+ * keeps rejecting content-free messages with 400.
+ */
+export const whatsappWebhookSchema = z
+  .object({
+    From: z.string().min(1, 'From is required'),
+    Body: z.string().optional(),
+    NumMedia: z.string().optional(),
+    MediaUrl0: z.string().url().optional(),
+    MediaContentType0: z.string().optional(),
+  })
+  .passthrough()
+  .refine((data) => Boolean(data.Body?.trim()) || Boolean(data.MediaUrl0), {
+    message: 'Either Body or media (MediaUrl0) is required',
+    path: ['Body'],
+  })
 
 const WEBHOOK_EVENTS = [
   'transaction.confirmed',

src/whatsapp/handler.ts

@@ -1,4 +1,4 @@
-import { parseIntent } from '../nlp/parser'
+import { parseIntent, type Intent } from '../nlp/parser'
 import { formatGoalProgressReply } from './formatters'
 import {
   normalizePhone,
@@ -11,11 +11,33 @@ import {
   getGoalStatus,
   decrementBalance,
 } from './userManager'
+import { logger } from '../utils/logger'
+import { config } from '../config'
+import { downloadTwilioMedia } from './mediaDownloader'
+import { getDefaultTranscriptionProvider } from './transcription/registry'
+import {
+  UnsupportedAudioError,
+  TranscriptionUnavailableError,
+} from './transcription/types'
+import {
+  getPendingConfirmation,
+  setPendingConfirmation,
+  clearPendingConfirmation,
+} from './pendingConfirmations'
 
 export type WhatsAppResponse = {
   body: string
 }
 
+/**
+ * Inbound media on a WhatsApp message (Twilio MediaUrl0 / MediaContentType0).
+ * Present only for voice notes and other attachments.
+ */
+export type InboundMedia = {
+  url: string
+  contentType: string
+}
+
 function formatHelpMessage(): string {
   return [
     'Welcome to NeuroWealth! Here are some things you can ask me:',
@@ -74,37 +96,57 @@ function extractOtpCode(message: string): string | null {
   return match ? match[1] : null
 }
 
-export async function handleWhatsAppMessage(
-  from: string,
-  message: string
-): Promise<WhatsAppResponse> {
-  const normalizedPhone = normalizePhone(from)
-  const user = await createOrGetUser(normalizedPhone)
+/**
+ * Financial / strategy-changing intents. When one of these originates from a
+ * voice note it must be confirmed before execution (#288) — misrecognition on
+ * a fund movement is materially worse than on a read-only query.
+ */
+const FINANCIAL_ACTIONS: ReadonlySet<Intent['action']> = new Set([
+  'deposit',
+  'withdraw',
+])
 
-  // If user is not verified, treat any 6-digit code as an OTP attempt.
-  if (!user.verified) {
-    const codeFromMessage = extractOtpCode(message)
-    if (codeFromMessage) {
-      const success = verifyOtp(normalizedPhone, codeFromMessage)
-      if (success) {
-        const wallet = getUserWalletAddress(normalizedPhone)
-        return {
-          body: `✅ Your account is now verified!\nYour wallet address is: ${wallet}\n\n${formatHelpMessage()}`,
-        }
-      }
+function isFinancialIntent(intent: Intent): boolean {
+  return FINANCIAL_ACTIONS.has(intent.action)
+}
 
-      return {
-        body: 'Invalid or expired OTP. Please request a new code by sending any message.',
-      }
-    }
+/** Affirmative reply to a pending confirmation ("yes", "confirm", "yeah"…). */
+export function isAffirmative(message: string): boolean {
+  return /^\s*(yes|yep|yeah|yup|confirm|confirmed|ok|okay|sure|correct|do it|go ahead|proceed|y)\s*[.!]*\s*$/i.test(
+    message
+  )
+}
 
-    // Send OTP for new user or re-send if not verified
-    const otp = generateOtp(normalizedPhone)
-    return { body: formatOtpMessage(otp) }
-  }
+/** Negative reply to a pending confirmation ("no", "cancel", "stop"…). */
+export function isNegative(message: string): boolean {
+  return /^\s*(no|nope|nah|cancel|stop|abort|don'?t|never mind|nevermind|n)\s*[.!]*\s*$/i.test(
+    message
+  )
+}
 
-  const intent = await parseIntent(message)
+/** Human-readable echo of a financial intent for the confirmation prompt. */
+export function summarizeIntent(intent: Intent): string {
+  switch (intent.action) {
+    case 'deposit':
+      return `deposit ${intent.amount ?? ''}`.trim()
+    case 'withdraw':
+      return intent.all
+        ? 'withdraw all'
+        : `withdraw ${intent.amount ?? ''}`.trim()
+    default:
+      return intent.action
+  }
+}
 
+/**
+ * Execute a parsed intent and produce the reply. This is the single place that
+ * acts on an intent, shared by typed messages, confirmed voice commands, and
+ * read-only voice commands — there is no voice-specific intent handling.
+ */
+async function executeIntent(
+  intent: Intent,
+  normalizedPhone: string
+): Promise<WhatsAppResponse> {
   switch (intent.action) {
     case 'balance': {
       const balance = getBalance(normalizedPhone) ?? 0
@@ -156,7 +198,6 @@ export async function handleWhatsAppMessage(
           body: 'I could not find any tracked portfolio data for your account yet.',
         }
       }
-
       return { body: formatEarnings(summary) }
     }
 
@@ -165,3 +206,138 @@ export async function handleWhatsAppMessage(
       return { body: formatUnknownMessage() }
   }
 }
+
+/**
+ * Resolve the text to act on for a message. For a voice note this downloads and
+ * transcribes the audio; for a text message it returns the body verbatim.
+ *
+ * Returns either { text, fromVoice } to proceed with parsing, or { reply } with
+ * a ready-made user-facing message when the voice note could not be used
+ * (unsupported format, low confidence, provider outage). Raw audio is discarded
+ * as soon as transcription returns — never persisted.
+ */
+async function resolveMessageText(
+  message: string,
+  media: InboundMedia | undefined
+): Promise<
+  { text: string; fromVoice: boolean } | { reply: string; fromVoice: boolean }
+> {
+  if (!media) {
+    return { text: message, fromVoice: false }
+  }
+
+  let audio
+  try {
+    audio = await downloadTwilioMedia(media.url, media.contentType)
+  } catch (err) {
+    return { reply: mediaErrorReply(err), fromVoice: true }
+  }
+
+  let result
+  try {
+    const provider = getDefaultTranscriptionProvider()
+    result = await provider.transcribe(audio)
+  } catch (err) {
+    return { reply: mediaErrorReply(err), fromVoice: true }
+  }
+
+  if (result.confidence < config.transcription.confidenceThreshold) {
+    logger.info(
+      `[Voice] Low-confidence transcription (${result.confidence.toFixed(2)} < ${config.transcription.confidenceThreshold}); asking user to repeat`
+    )
+    return {
+      reply:
+        "I couldn't quite catch that. Please repeat your voice message or type your command.",
+      fromVoice: true,
+    }
+  }
+
+  return { text: result.text, fromVoice: true }
+}
+
+/** Map a media/transcription error onto the right user-facing fallback. */
+function mediaErrorReply(err: unknown): string {
+  if (err instanceof UnsupportedAudioError) {
+    return "I can't process that audio format. Please try again or type your command."
+  }
+  if (err instanceof TranscriptionUnavailableError) {
+    logger.warn(`[Voice] Transcription unavailable: ${err.message}`)
+    return "Voice messages aren't available right now. Please type your command."
+  }
+  logger.error('[Voice] Unexpected error handling voice note', {
+    error: err instanceof Error ? err.message : String(err),
+  })
+  return "Voice messages aren't available right now. Please type your command."
+}
+
+export async function handleWhatsAppMessage(
+  from: string,
+  message: string,
+  media?: InboundMedia
+): Promise<WhatsAppResponse> {
+  const normalizedPhone = normalizePhone(from)
+  const user = await createOrGetUser(normalizedPhone)
+
+  // If user is not verified, treat any 6-digit code as an OTP attempt.
+  // OTP is text-only; a voice note here still triggers the resend path below.
+  if (!user.verified) {
+    const codeFromMessage = extractOtpCode(message)
+    if (codeFromMessage) {
+      const success = verifyOtp(normalizedPhone, codeFromMessage)
+      if (success) {
+        const wallet = getUserWalletAddress(normalizedPhone)
+        return {
+          body: `✅ Your account is now verified!\nYour wallet address is: ${wallet}\n\n${formatHelpMessage()}`,
+        }
+      }
+
+      return {
+        body: 'Invalid or expired OTP. Please request a new code by sending any message.',
+      }
+    }
+
+    // Send OTP for new user or re-send if not verified
+    const otp = generateOtp(normalizedPhone)
+    return { body: formatOtpMessage(otp) }
+  }
+
+  // Resolve the text to act on — transcribing first if this is a voice note.
+  const resolved = await resolveMessageText(message, media)
+  if ('reply' in resolved) {
+    return { body: resolved.reply }
+  }
+  const { text, fromVoice } = resolved
+
+  // If a voice-originated financial action is awaiting confirmation, the next
+  // message (voice OR text) is interpreted as the yes/no reply.
+  const pending = getPendingConfirmation(normalizedPhone)
+  if (pending) {
+    if (isAffirmative(text)) {
+      clearPendingConfirmation(normalizedPhone)
+      return executeIntent(pending.intent, normalizedPhone)
+    }
+    if (isNegative(text)) {
+      clearPendingConfirmation(normalizedPhone)
+      return { body: 'Okay, cancelled. Nothing was done.' }
+    }
+    // Anything else: keep the pending action and re-prompt rather than
+    // silently dropping it or acting on the new message.
+    return {
+      body: `You still have a pending action: ${pending.summary}. Reply "yes" to confirm or "no" to cancel.`,
+    }
+  }
+
+  const intent = await parseIntent(text)
+
+  // Confirm-before-execute for financial intents that came from voice. This is
+  // a security control against misrecognition and must not be bypassable.
+  if (fromVoice && isFinancialIntent(intent)) {
+    const summary = summarizeIntent(intent)
+    setPendingConfirmation(normalizedPhone, intent, summary)
+    return {
+      body: `I heard: *${summary}*.\nReply "yes" to confirm or "no" to cancel.`,
+    }
+  }
+
+  return executeIntent(intent, normalizedPhone)
+}