diff --git a/src/config/env.ts b/src/config/env.ts index d5bd4f2..9101a76 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -420,6 +420,25 @@ export const config = { twilioToken: process.env.TWILIO_AUTH_TOKEN || '', fromNumber: process.env.WHATSAPP_FROM || '', }, + // Speech-to-text for WhatsApp voice notes (#288). The provider is swappable + // behind the TranscriptionProvider interface; these settings configure the + // default (OpenAI Whisper) implementation. Raw audio is never persisted — + // see docs/WHATSAPP_VOICE.md. + transcription: { + provider: process.env.TRANSCRIPTION_PROVIDER || 'openai', + openaiApiKey: process.env.OPENAI_API_KEY || '', + model: process.env.TRANSCRIPTION_MODEL || 'whisper-1', + apiUrl: + process.env.TRANSCRIPTION_API_URL || + 'https://api.openai.com/v1/audio/transcriptions', + /** + * Minimum transcription confidence (0–1) required to act on a voice note. + * Below this the bot asks the user to repeat or type rather than guessing. + */ + confidenceThreshold: parseFloat( + process.env.TRANSCRIPTION_CONFIDENCE_THRESHOLD || '0.6' + ), + }, dlq: { alertThreshold: parseInt(process.env.DLQ_ALERT_THRESHOLD || '50'), alertCooldownMs: parseInt(process.env.DLQ_ALERT_COOLDOWN_MS || '900000'), // 15 minutes default diff --git a/src/routes/whatsapp.ts b/src/routes/whatsapp.ts index 23dcf02..5c15615 100644 --- a/src/routes/whatsapp.ts +++ b/src/routes/whatsapp.ts @@ -65,8 +65,22 @@ router.post( const from = (req.body.From as string) || '' const body = (req.body.Body as string) || '' + // Voice notes (and other media) arrive with NumMedia > 0 and MediaUrl0 / + // MediaContentType0. We only act on audio; non-audio media falls through to + // text handling (empty body → 'unknown'). + const numMedia = parseInt((req.body.NumMedia as string) || '0', 10) + const mediaUrl = req.body.MediaUrl0 as string | undefined + const mediaContentType = req.body.MediaContentType0 as string | undefined + const media = + numMedia > 0 && + mediaUrl && + mediaContentType && + mediaContentType.startsWith('audio/') + ? { url: mediaUrl, contentType: mediaContentType } + : undefined + try { - const response = await handleWhatsAppMessage(from, body) + const response = await handleWhatsAppMessage(from, body, media) const responseTwiml = new twiml.MessagingResponse() responseTwiml.message(response.body) res.type('text/xml').send(responseTwiml.toString()) diff --git a/src/validators/webhook-validators.ts b/src/validators/webhook-validators.ts index e453341..966ded5 100644 --- a/src/validators/webhook-validators.ts +++ b/src/validators/webhook-validators.ts @@ -1,9 +1,30 @@ import { z } from 'zod' -export const whatsappWebhookSchema = z.object({ - From: z.string().min(1, 'From is required'), - Body: z.string().min(1, 'Body is required'), -}) +/** + * Twilio inbound WhatsApp webhook. `Body` is optional because a voice note + * arrives with media fields and an empty/absent body (#288). `NumMedia`/ + * `MediaUrl0`/`MediaContentType0` are Twilio's media fields; we read the first + * attachment. `.passthrough()` keeps Twilio's other fields and — crucially — we + * add NO defaulted keys, so the object still matches the exact params Twilio + * signed (signature validation runs on req.body after this parse). + * + * A message must carry either text or media: `Body` alone being optional would + * let an empty payload through to the signature check, so the refinement below + * keeps rejecting content-free messages with 400. + */ +export const whatsappWebhookSchema = z + .object({ + From: z.string().min(1, 'From is required'), + Body: z.string().optional(), + NumMedia: z.string().optional(), + MediaUrl0: z.string().url().optional(), + MediaContentType0: z.string().optional(), + }) + .passthrough() + .refine((data) => Boolean(data.Body?.trim()) || Boolean(data.MediaUrl0), { + message: 'Either Body or media (MediaUrl0) is required', + path: ['Body'], + }) const WEBHOOK_EVENTS = [ 'transaction.confirmed', diff --git a/src/whatsapp/handler.ts b/src/whatsapp/handler.ts index d80415b..e3f03b1 100644 --- a/src/whatsapp/handler.ts +++ b/src/whatsapp/handler.ts @@ -1,4 +1,4 @@ -import { parseIntent } from '../nlp/parser' +import { parseIntent, type Intent } from '../nlp/parser' import { formatGoalProgressReply } from './formatters' import { normalizePhone, @@ -11,11 +11,33 @@ import { getGoalStatus, decrementBalance, } from './userManager' +import { logger } from '../utils/logger' +import { config } from '../config' +import { downloadTwilioMedia } from './mediaDownloader' +import { getDefaultTranscriptionProvider } from './transcription/registry' +import { + UnsupportedAudioError, + TranscriptionUnavailableError, +} from './transcription/types' +import { + getPendingConfirmation, + setPendingConfirmation, + clearPendingConfirmation, +} from './pendingConfirmations' export type WhatsAppResponse = { body: string } +/** + * Inbound media on a WhatsApp message (Twilio MediaUrl0 / MediaContentType0). + * Present only for voice notes and other attachments. + */ +export type InboundMedia = { + url: string + contentType: string +} + function formatHelpMessage(): string { return [ 'Welcome to NeuroWealth! Here are some things you can ask me:', @@ -74,37 +96,57 @@ function extractOtpCode(message: string): string | null { return match ? match[1] : null } -export async function handleWhatsAppMessage( - from: string, - message: string -): Promise { - const normalizedPhone = normalizePhone(from) - const user = await createOrGetUser(normalizedPhone) +/** + * Financial / strategy-changing intents. When one of these originates from a + * voice note it must be confirmed before execution (#288) — misrecognition on + * a fund movement is materially worse than on a read-only query. + */ +const FINANCIAL_ACTIONS: ReadonlySet = new Set([ + 'deposit', + 'withdraw', +]) - // If user is not verified, treat any 6-digit code as an OTP attempt. - if (!user.verified) { - const codeFromMessage = extractOtpCode(message) - if (codeFromMessage) { - const success = verifyOtp(normalizedPhone, codeFromMessage) - if (success) { - const wallet = getUserWalletAddress(normalizedPhone) - return { - body: `✅ Your account is now verified!\nYour wallet address is: ${wallet}\n\n${formatHelpMessage()}`, - } - } +function isFinancialIntent(intent: Intent): boolean { + return FINANCIAL_ACTIONS.has(intent.action) +} - return { - body: 'Invalid or expired OTP. Please request a new code by sending any message.', - } - } +/** Affirmative reply to a pending confirmation ("yes", "confirm", "yeah"…). */ +export function isAffirmative(message: string): boolean { + return /^\s*(yes|yep|yeah|yup|confirm|confirmed|ok|okay|sure|correct|do it|go ahead|proceed|y)\s*[.!]*\s*$/i.test( + message + ) +} - // Send OTP for new user or re-send if not verified - const otp = generateOtp(normalizedPhone) - return { body: formatOtpMessage(otp) } - } +/** Negative reply to a pending confirmation ("no", "cancel", "stop"…). */ +export function isNegative(message: string): boolean { + return /^\s*(no|nope|nah|cancel|stop|abort|don'?t|never mind|nevermind|n)\s*[.!]*\s*$/i.test( + message + ) +} - const intent = await parseIntent(message) +/** Human-readable echo of a financial intent for the confirmation prompt. */ +export function summarizeIntent(intent: Intent): string { + switch (intent.action) { + case 'deposit': + return `deposit ${intent.amount ?? ''}`.trim() + case 'withdraw': + return intent.all + ? 'withdraw all' + : `withdraw ${intent.amount ?? ''}`.trim() + default: + return intent.action + } +} +/** + * Execute a parsed intent and produce the reply. This is the single place that + * acts on an intent, shared by typed messages, confirmed voice commands, and + * read-only voice commands — there is no voice-specific intent handling. + */ +async function executeIntent( + intent: Intent, + normalizedPhone: string +): Promise { switch (intent.action) { case 'balance': { const balance = getBalance(normalizedPhone) ?? 0 @@ -156,7 +198,6 @@ export async function handleWhatsAppMessage( body: 'I could not find any tracked portfolio data for your account yet.', } } - return { body: formatEarnings(summary) } } @@ -165,3 +206,138 @@ export async function handleWhatsAppMessage( return { body: formatUnknownMessage() } } } + +/** + * Resolve the text to act on for a message. For a voice note this downloads and + * transcribes the audio; for a text message it returns the body verbatim. + * + * Returns either { text, fromVoice } to proceed with parsing, or { reply } with + * a ready-made user-facing message when the voice note could not be used + * (unsupported format, low confidence, provider outage). Raw audio is discarded + * as soon as transcription returns — never persisted. + */ +async function resolveMessageText( + message: string, + media: InboundMedia | undefined +): Promise< + { text: string; fromVoice: boolean } | { reply: string; fromVoice: boolean } +> { + if (!media) { + return { text: message, fromVoice: false } + } + + let audio + try { + audio = await downloadTwilioMedia(media.url, media.contentType) + } catch (err) { + return { reply: mediaErrorReply(err), fromVoice: true } + } + + let result + try { + const provider = getDefaultTranscriptionProvider() + result = await provider.transcribe(audio) + } catch (err) { + return { reply: mediaErrorReply(err), fromVoice: true } + } + + if (result.confidence < config.transcription.confidenceThreshold) { + logger.info( + `[Voice] Low-confidence transcription (${result.confidence.toFixed(2)} < ${config.transcription.confidenceThreshold}); asking user to repeat` + ) + return { + reply: + "I couldn't quite catch that. Please repeat your voice message or type your command.", + fromVoice: true, + } + } + + return { text: result.text, fromVoice: true } +} + +/** Map a media/transcription error onto the right user-facing fallback. */ +function mediaErrorReply(err: unknown): string { + if (err instanceof UnsupportedAudioError) { + return "I can't process that audio format. Please try again or type your command." + } + if (err instanceof TranscriptionUnavailableError) { + logger.warn(`[Voice] Transcription unavailable: ${err.message}`) + return "Voice messages aren't available right now. Please type your command." + } + logger.error('[Voice] Unexpected error handling voice note', { + error: err instanceof Error ? err.message : String(err), + }) + return "Voice messages aren't available right now. Please type your command." +} + +export async function handleWhatsAppMessage( + from: string, + message: string, + media?: InboundMedia +): Promise { + const normalizedPhone = normalizePhone(from) + const user = await createOrGetUser(normalizedPhone) + + // If user is not verified, treat any 6-digit code as an OTP attempt. + // OTP is text-only; a voice note here still triggers the resend path below. + if (!user.verified) { + const codeFromMessage = extractOtpCode(message) + if (codeFromMessage) { + const success = verifyOtp(normalizedPhone, codeFromMessage) + if (success) { + const wallet = getUserWalletAddress(normalizedPhone) + return { + body: `✅ Your account is now verified!\nYour wallet address is: ${wallet}\n\n${formatHelpMessage()}`, + } + } + + return { + body: 'Invalid or expired OTP. Please request a new code by sending any message.', + } + } + + // Send OTP for new user or re-send if not verified + const otp = generateOtp(normalizedPhone) + return { body: formatOtpMessage(otp) } + } + + // Resolve the text to act on — transcribing first if this is a voice note. + const resolved = await resolveMessageText(message, media) + if ('reply' in resolved) { + return { body: resolved.reply } + } + const { text, fromVoice } = resolved + + // If a voice-originated financial action is awaiting confirmation, the next + // message (voice OR text) is interpreted as the yes/no reply. + const pending = getPendingConfirmation(normalizedPhone) + if (pending) { + if (isAffirmative(text)) { + clearPendingConfirmation(normalizedPhone) + return executeIntent(pending.intent, normalizedPhone) + } + if (isNegative(text)) { + clearPendingConfirmation(normalizedPhone) + return { body: 'Okay, cancelled. Nothing was done.' } + } + // Anything else: keep the pending action and re-prompt rather than + // silently dropping it or acting on the new message. + return { + body: `You still have a pending action: ${pending.summary}. Reply "yes" to confirm or "no" to cancel.`, + } + } + + const intent = await parseIntent(text) + + // Confirm-before-execute for financial intents that came from voice. This is + // a security control against misrecognition and must not be bypassable. + if (fromVoice && isFinancialIntent(intent)) { + const summary = summarizeIntent(intent) + setPendingConfirmation(normalizedPhone, intent, summary) + return { + body: `I heard: *${summary}*.\nReply "yes" to confirm or "no" to cancel.`, + } + } + + return executeIntent(intent, normalizedPhone) +} diff --git a/src/whatsapp/mediaDownloader.ts b/src/whatsapp/mediaDownloader.ts new file mode 100644 index 0000000..584c0f3 --- /dev/null +++ b/src/whatsapp/mediaDownloader.ts @@ -0,0 +1,98 @@ +import { config } from '../config' +import { logger } from '../utils/logger' +import { + TranscriptionUnavailableError, + UnsupportedAudioError, + isSupportedAudioType, +} from './transcription/types' + +/** + * Download an inbound WhatsApp media attachment from Twilio (#288). + * + * Twilio media URLs are protected by HTTP basic auth using the account SID and + * auth token. We fetch the bytes into memory only — the buffer is handed to the + * transcription provider and then discarded; nothing is written to disk (see + * docs/WHATSAPP_VOICE.md). + * + * Twilio often 302-redirects the MediaUrl to a signed CDN URL; global fetch + * follows redirects by default, and the CDN response carries the real + * Content-Type, which we trust over the webhook's MediaContentType field. + */ + +export interface DownloadedMedia { + buffer: Buffer + contentType: string +} + +/** Max audio we will pull into memory. WhatsApp voice notes are far smaller. */ +const MAX_MEDIA_BYTES = 16 * 1024 * 1024 // 16 MB + +export async function downloadTwilioMedia( + mediaUrl: string, + declaredContentType: string +): Promise { + // Reject clearly-unsupported formats before spending a network round-trip. + // The authoritative check happens again on the downloaded Content-Type. + if (declaredContentType && !isSupportedAudioType(declaredContentType)) { + throw new UnsupportedAudioError( + `Unsupported audio content type: ${declaredContentType}` + ) + } + + const sid = config.whatsapp.twilioSid + const token = config.whatsapp.twilioToken + if (!sid || !token) { + throw new TranscriptionUnavailableError( + 'Twilio credentials not configured for media download' + ) + } + + const auth = Buffer.from(`${sid}:${token}`).toString('base64') + + let response: Response + try { + const controller = new AbortController() + const timer = setTimeout( + () => controller.abort(), + config.httpClient.timeoutMs + ) + try { + response = await fetch(mediaUrl, { + headers: { Authorization: `Basic ${auth}` }, + signal: controller.signal, + }) + } finally { + clearTimeout(timer) + } + } catch (err) { + throw new TranscriptionUnavailableError( + `Media download failed: ${err instanceof Error ? err.message : String(err)}` + ) + } + + if (!response.ok) { + logger.warn(`[Media] Twilio media download HTTP ${response.status}`) + throw new TranscriptionUnavailableError( + `Media download error (HTTP ${response.status})` + ) + } + + const resolvedType = + response.headers.get('content-type')?.split(';')[0]?.trim() || + declaredContentType + + if (!isSupportedAudioType(resolvedType)) { + throw new UnsupportedAudioError( + `Unsupported audio content type: ${resolvedType}` + ) + } + + const arrayBuffer = await response.arrayBuffer() + if (arrayBuffer.byteLength > MAX_MEDIA_BYTES) { + throw new UnsupportedAudioError( + `Audio is too large (${arrayBuffer.byteLength} bytes)` + ) + } + + return { buffer: Buffer.from(arrayBuffer), contentType: resolvedType } +} diff --git a/src/whatsapp/pendingConfirmations.ts b/src/whatsapp/pendingConfirmations.ts new file mode 100644 index 0000000..e9c0054 --- /dev/null +++ b/src/whatsapp/pendingConfirmations.ts @@ -0,0 +1,61 @@ +import type { Intent } from '../nlp/parser' + +/** + * Pending voice-originated confirmations (#288). + * + * A financial intent (deposit/withdraw/strategy change) recognized from a voice + * note is NOT executed immediately — it is parked here and echoed back to the + * user ("I heard: withdraw $50 — confirm?"). The user's next message (voice or + * text) is checked for an affirmative/negative reply before the intent runs. + * + * In-memory and keyed by normalized phone, matching the existing WhatsApp user + * store. A pending confirmation expires after a short TTL so a stale "yes" long + * after the fact can never trigger a fund movement. + */ + +export interface PendingConfirmation { + intent: Intent + /** Human-readable echo shown to the user, kept for logging/debugging. */ + summary: string + expiresAt: number +} + +const store = new Map() + +/** How long a parked confirmation stays valid. */ +export const CONFIRMATION_TTL_MS = 5 * 60 * 1000 // 5 minutes + +export function setPendingConfirmation( + phone: string, + intent: Intent, + summary: string, + now: number = Date.now() +): void { + store.set(phone, { intent, summary, expiresAt: now + CONFIRMATION_TTL_MS }) +} + +/** + * Return the live pending confirmation for a phone, or null if none exists or + * it has expired. Expired entries are evicted on read. + */ +export function getPendingConfirmation( + phone: string, + now: number = Date.now() +): PendingConfirmation | null { + const pending = store.get(phone) + if (!pending) return null + if (now >= pending.expiresAt) { + store.delete(phone) + return null + } + return pending +} + +export function clearPendingConfirmation(phone: string): void { + store.delete(phone) +} + +/** Test seam. */ +export function clearAllPendingConfirmations(): void { + store.clear() +} diff --git a/src/whatsapp/transcription/openaiProvider.ts b/src/whatsapp/transcription/openaiProvider.ts new file mode 100644 index 0000000..c9d1f01 --- /dev/null +++ b/src/whatsapp/transcription/openaiProvider.ts @@ -0,0 +1,154 @@ +import { config } from '../../config' +import { logger } from '../../utils/logger' +import { + AudioInput, + TranscriptionProvider, + TranscriptionResult, + TranscriptionUnavailableError, + UnsupportedAudioError, + isSupportedAudioType, +} from './types' + +/** + * OpenAI Whisper transcription provider (#288). + * + * The one concrete {@link TranscriptionProvider} shipped in v1. Uses the + * `verbose_json` response so we can derive a confidence score from segment + * average log-probabilities (Whisper does not return a single top-level + * confidence). Talks to the HTTP API directly via global fetch/FormData + * (Node 18+) so no vendor SDK is pulled in. + * + * Privacy: the audio buffer is held only for the duration of the request and + * is never written to disk. See docs/WHATSAPP_VOICE.md. + */ + +interface WhisperSegment { + avg_logprob?: number +} + +interface WhisperVerboseResponse { + text?: string + segments?: WhisperSegment[] +} + +/** Map a filename extension onto the content type for the multipart part. */ +function filenameFor(contentType: string): string { + const base = contentType.split(';')[0]?.trim().toLowerCase() + const ext = + base === 'audio/mpeg' || base === 'audio/mp3' + ? 'mp3' + : base === 'audio/mp4' + ? 'm4a' + : base === 'audio/wav' || base === 'audio/x-wav' + ? 'wav' + : base === 'audio/webm' + ? 'webm' + : base === 'audio/amr' + ? 'amr' + : 'ogg' + return `voice-note.${ext}` +} + +/** + * Convert Whisper's mean segment log-probability into a [0,1] confidence. + * exp(avg_logprob) is the geometric-mean token probability, which is a + * reasonable, monotonic proxy for "how sure was the model". Clamped to [0,1]. + */ +export function confidenceFromSegments(segments: WhisperSegment[]): number { + const logprobs = segments + .map((s) => s.avg_logprob) + .filter((v): v is number => typeof v === 'number' && Number.isFinite(v)) + + if (logprobs.length === 0) { + // No per-segment data (e.g. very short clip). Treat as mid confidence so a + // clear short command isn't force-rejected, but a garbled one still can be + // caught by the parser falling through to 'unknown'. + return 0.7 + } + + const mean = logprobs.reduce((a, b) => a + b, 0) / logprobs.length + const conf = Math.exp(mean) + return Math.max(0, Math.min(1, conf)) +} + +export class OpenAiTranscriptionProvider implements TranscriptionProvider { + readonly name = 'openai' + + async transcribe(audio: AudioInput): Promise { + if (!isSupportedAudioType(audio.contentType)) { + throw new UnsupportedAudioError( + `Unsupported audio content type: ${audio.contentType}` + ) + } + + const apiKey = config.transcription.openaiApiKey + if (!apiKey) { + // Missing credentials is an availability problem from the user's POV. + throw new TranscriptionUnavailableError( + 'Transcription provider is not configured (missing OPENAI_API_KEY)' + ) + } + + const form = new FormData() + const blob = new Blob([audio.buffer], { + type: audio.contentType.split(';')[0]?.trim() || 'audio/ogg', + }) + form.append('file', blob, filenameFor(audio.contentType)) + form.append('model', config.transcription.model) + form.append('response_format', 'verbose_json') + + let response: Response + try { + const controller = new AbortController() + const timer = setTimeout( + () => controller.abort(), + config.httpClient.timeoutMs + ) + try { + response = await fetch(config.transcription.apiUrl, { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}` }, + body: form, + signal: controller.signal, + }) + } finally { + clearTimeout(timer) + } + } catch (err) { + // Network error, DNS failure, timeout/abort — provider is unavailable. + throw new TranscriptionUnavailableError( + `Transcription request failed: ${err instanceof Error ? err.message : String(err)}` + ) + } + + if (!response.ok) { + const detail = await response.text().catch(() => '') + // 4xx on the audio itself (e.g. unprocessable format) → unsupported; + // everything else (5xx, 429, auth) → unavailable/outage. + if (response.status === 400 || response.status === 415) { + throw new UnsupportedAudioError( + `Provider rejected the audio (HTTP ${response.status})` + ) + } + logger.warn( + `[Transcription] OpenAI returned HTTP ${response.status}: ${detail.slice(0, 200)}` + ) + throw new TranscriptionUnavailableError( + `Transcription provider error (HTTP ${response.status})` + ) + } + + let payload: WhisperVerboseResponse + try { + payload = (await response.json()) as WhisperVerboseResponse + } catch (err) { + throw new TranscriptionUnavailableError( + `Could not parse transcription response: ${err instanceof Error ? err.message : String(err)}` + ) + } + + const text = (payload.text ?? '').trim() + const confidence = confidenceFromSegments(payload.segments ?? []) + return { text, confidence } + } +} diff --git a/src/whatsapp/transcription/registry.ts b/src/whatsapp/transcription/registry.ts new file mode 100644 index 0000000..7407ba1 --- /dev/null +++ b/src/whatsapp/transcription/registry.ts @@ -0,0 +1,45 @@ +/** + * Transcription provider registry (#288). + * + * The single lookup point mapping a provider key to a + * {@link TranscriptionProvider}. The WhatsApp handler resolves the active + * provider exclusively through {@link getDefaultTranscriptionProvider}, so + * swapping STT vendors is a one-line registry change (plus config) with no edits + * to the handler — the same pattern as the fiat provider registry. + */ +import { config } from '../../config' +import { TranscriptionProvider } from './types' +import { OpenAiTranscriptionProvider } from './openaiProvider' + +const registry = new Map() + +function register(provider: TranscriptionProvider): void { + registry.set(provider.name, provider) +} + +// v1 ships a single provider. Add further vendors here — nothing else changes. +register(new OpenAiTranscriptionProvider()) + +/** Resolve a provider by key. Throws if the key is unknown/unconfigured. */ +export function getTranscriptionProvider(name: string): TranscriptionProvider { + const provider = registry.get(name) + if (!provider) { + throw new Error(`Unknown transcription provider: "${name}"`) + } + return provider +} + +/** The active provider for incoming voice notes (from config). */ +export function getDefaultTranscriptionProvider(): TranscriptionProvider { + return getTranscriptionProvider(config.transcription.provider) +} + +/** + * Test/bootstrap seam: replace or add a provider implementation without going + * through env configuration. Used by unit tests to inject a stub. + */ +export function registerTranscriptionProvider( + provider: TranscriptionProvider +): void { + register(provider) +} diff --git a/src/whatsapp/transcription/types.ts b/src/whatsapp/transcription/types.ts new file mode 100644 index 0000000..47f619d --- /dev/null +++ b/src/whatsapp/transcription/types.ts @@ -0,0 +1,92 @@ +/** + * Speech-to-text abstraction for WhatsApp voice notes (#288). + * + * The concrete STT vendor MUST live behind {@link TranscriptionProvider} so it + * is swappable and never hard-coded into the WhatsApp handler — mirroring how + * the Stellar RPC client sits behind ResilientRpcClient and fiat vendors sit + * behind FiatRampProvider. Nothing outside `src/whatsapp/transcription/*` + * should reference a specific STT vendor. + * + * Privacy: implementations receive an in-memory audio buffer and MUST NOT + * persist it. The raw audio is discarded once transcription returns. See + * docs/WHATSAPP_VOICE.md. + */ + +export interface TranscriptionResult { + /** The recognized text. May be empty if the audio contained no speech. */ + text: string + /** + * Confidence in [0, 1]. Providers that don't natively return a single score + * should derive a reasonable proxy (e.g. from segment log-probabilities) so + * the low-confidence gate in the handler has something to compare against. + */ + confidence: number +} + +export interface AudioInput { + /** Raw audio bytes downloaded from the messaging provider. */ + buffer: Buffer + /** MIME type reported by the messaging provider (e.g. "audio/ogg"). */ + contentType: string +} + +export interface TranscriptionProvider { + /** Stable key (e.g. "openai"), used by the registry and logs. */ + readonly name: string + /** + * Transcribe a complete audio note. Implementations should throw + * {@link UnsupportedAudioError} for a format they cannot handle and + * {@link TranscriptionUnavailableError} for an outage/transport failure, so + * the handler can produce the right user-facing message for each. + */ + transcribe(audio: AudioInput): Promise +} + +/** + * The audio format/codec is one the provider cannot process. Distinct from an + * outage so the handler can tell the user their audio wasn't understood rather + * than that the service is down. + */ +export class UnsupportedAudioError extends Error { + constructor(message: string) { + super(message) + this.name = 'UnsupportedAudioError' + Object.setPrototypeOf(this, UnsupportedAudioError.prototype) + } +} + +/** + * The provider is unreachable or returned a transport-level failure. Signals + * the "voice isn't available right now, please type" fallback. + */ +export class TranscriptionUnavailableError extends Error { + constructor(message: string) { + super(message) + this.name = 'TranscriptionUnavailableError' + Object.setPrototypeOf(this, TranscriptionUnavailableError.prototype) + } +} + +/** + * Audio MIME types we accept before ever calling the provider. WhatsApp voice + * notes are delivered as OGG/Opus; the others are accepted defensively for + * clients that transcode. Anything else is rejected up front with a graceful + * message rather than sent to the provider to fail opaquely. + */ +export const SUPPORTED_AUDIO_MIME_TYPES = [ + 'audio/ogg', + 'audio/opus', + 'audio/mpeg', + 'audio/mp4', + 'audio/mp3', + 'audio/amr', + 'audio/wav', + 'audio/x-wav', + 'audio/webm', +] as const + +/** True when `contentType` (ignoring any `;codecs=` suffix) is one we accept. */ +export function isSupportedAudioType(contentType: string): boolean { + const base = contentType.split(';')[0]?.trim().toLowerCase() + return (SUPPORTED_AUDIO_MIME_TYPES as readonly string[]).includes(base ?? '') +} diff --git a/tests/fixtures/audio/unsupported.bin b/tests/fixtures/audio/unsupported.bin new file mode 100644 index 0000000..d5c5c7d --- /dev/null +++ b/tests/fixtures/audio/unsupported.bin @@ -0,0 +1 @@ +not-audio-just-bytes \ No newline at end of file diff --git a/tests/fixtures/audio/voice-note.ogg b/tests/fixtures/audio/voice-note.ogg new file mode 100644 index 0000000..fcc73b2 Binary files /dev/null and b/tests/fixtures/audio/voice-note.ogg differ diff --git a/tests/unit/whatsapp/voiceHandler.test.ts b/tests/unit/whatsapp/voiceHandler.test.ts new file mode 100644 index 0000000..1eb9049 --- /dev/null +++ b/tests/unit/whatsapp/voiceHandler.test.ts @@ -0,0 +1,206 @@ +// WhatsApp voice-note handling (#288). Fixture audio stands in for real bytes; +// the media download and STT provider are mocked so these tests are +// deterministic and offline. The security-critical assertion is that a +// voice-originated financial intent NEVER executes on the first pass. +process.env.NODE_ENV = 'test' +process.env.STELLAR_NETWORK = 'testnet' +process.env.STELLAR_RPC_URL = 'https://soroban-testnet.stellar.org' +process.env.STELLAR_AGENT_SECRET_KEY = 'S' + 'A'.repeat(55) +process.env.VAULT_CONTRACT_ID = 'C' + 'A'.repeat(55) +process.env.USDC_TOKEN_ADDRESS = 'C' + 'B'.repeat(55) +process.env.ANTHROPIC_API_KEY = 'sk-ant-test-key' +process.env.DATABASE_URL = 'postgresql://localhost:5432/test' +process.env.JWT_SEED = '0'.repeat(64) +process.env.WALLET_ENCRYPTION_KEY = '0'.repeat(64) +process.env.TWILIO_AUTH_TOKEN = '0'.repeat(32) +process.env.TWILIO_ACCOUNT_SID = 'AC' + '0'.repeat(32) +// Force the regex fast-path only, so no live Claude call is attempted. +process.env.AI_MODE = 'local' + +import fs from 'fs' +import path from 'path' + +import { downloadTwilioMedia } from '../../../src/whatsapp/mediaDownloader' +import { + UnsupportedAudioError, + TranscriptionUnavailableError, + type TranscriptionProvider, +} from '../../../src/whatsapp/transcription/types' +import { registerTranscriptionProvider } from '../../../src/whatsapp/transcription/registry' + +jest.mock('../../../src/whatsapp/mediaDownloader', () => ({ + downloadTwilioMedia: jest.fn(), +})) +jest.mock('../../../src/utils/logger', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})) + +// Stub the custodial-wallet creation the user store triggers, so tests don't +// touch Stellar/crypto. +jest.mock('../../../src/stellar/wallet', () => ({ + createCustodialWallet: jest + .fn() + .mockResolvedValue({ publicKey: 'G' + 'A'.repeat(55) }), + getWalletByUserId: jest + .fn() + .mockResolvedValue({ publicKey: 'G' + 'A'.repeat(55) }), +})) + +import { handleWhatsAppMessage } from '../../../src/whatsapp/handler' +import { + getUserForTests, + clearUsersForTests, +} from '../../../src/whatsapp/userManager' +import { clearAllPendingConfirmations } from '../../../src/whatsapp/pendingConfirmations' + +const mockDownload = downloadTwilioMedia as jest.Mock + +const FIXTURE = fs.readFileSync( + path.join(__dirname, '../../fixtures/audio/voice-note.ogg') +) + +const PHONE = 'whatsapp:+15551234567' +const MEDIA = { + url: 'https://api.twilio.com/media/abc', + contentType: 'audio/ogg', +} + +/** Register a one-off transcription provider returning the given result/throw. */ +function useProvider( + impl: (buf: Buffer) => Promise<{ text: string; confidence: number }> +): void { + const provider: TranscriptionProvider = { + name: 'openai', // same key as the default, so registry lookup resolves to this + transcribe: (audio) => impl(audio.buffer), + } + registerTranscriptionProvider(provider) +} + +/** Verify + fund a user so financial intents can pass balance checks. */ +async function verifiedUser(phone: string, balance = 1000): Promise { + await handleWhatsAppMessage(phone, 'hello') // creates user, sends OTP + const user = getUserForTests(phone)! + // Directly flip verification + balance via the test view. + ;(user as any).verified = true + ;(user as any).balance = balance +} + +beforeEach(() => { + jest.clearAllMocks() + clearUsersForTests() + clearAllPendingConfirmations() + mockDownload.mockResolvedValue({ buffer: FIXTURE, contentType: 'audio/ogg' }) +}) + +describe('WhatsApp voice notes (#288)', () => { + it('transcribes a clear read-only command and executes it directly', async () => { + await verifiedUser(PHONE) + useProvider(async () => ({ text: 'balance', confidence: 0.95 })) + + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + expect(mockDownload).toHaveBeenCalledWith(MEDIA.url, MEDIA.contentType) + expect(res.body).toMatch(/balance/i) + }) + + it('does NOT execute a voice-originated withdraw on the first pass — asks to confirm', async () => { + await verifiedUser(PHONE, 500) + useProvider(async () => ({ text: 'withdraw 50', confidence: 0.95 })) + + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + // Confirmation prompt, NOT a withdrawal confirmation. + expect(res.body).toMatch(/confirm/i) + expect(res.body).toMatch(/withdraw 50/i) + // Balance untouched — nothing executed. + expect(getUserForTests(PHONE)!.balance).toBe(500) + }) + + it('executes the withdraw only after an affirmative reply (text)', async () => { + await verifiedUser(PHONE, 500) + useProvider(async () => ({ text: 'withdraw 50', confidence: 0.95 })) + + await handleWhatsAppMessage(PHONE, '', MEDIA) // parks confirmation + const res = await handleWhatsAppMessage(PHONE, 'yes') // text confirm + + expect(res.body).toMatch(/withdrawal request received/i) + expect(getUserForTests(PHONE)!.balance).toBe(450) + }) + + it('accepts a voice "yes" as confirmation of a voice-originated action', async () => { + await verifiedUser(PHONE, 500) + useProvider(async () => ({ text: 'withdraw 50', confidence: 0.95 })) + await handleWhatsAppMessage(PHONE, '', MEDIA) // parks confirmation + + // Confirmation arrives as a voice note saying "yes". + useProvider(async () => ({ text: 'yes', confidence: 0.95 })) + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + expect(res.body).toMatch(/withdrawal request received/i) + expect(getUserForTests(PHONE)!.balance).toBe(450) + }) + + it('cancels the pending action on a negative reply without executing', async () => { + await verifiedUser(PHONE, 500) + useProvider(async () => ({ text: 'withdraw 50', confidence: 0.95 })) + await handleWhatsAppMessage(PHONE, '', MEDIA) + + const res = await handleWhatsAppMessage(PHONE, 'no') + + expect(res.body).toMatch(/cancel/i) + expect(getUserForTests(PHONE)!.balance).toBe(500) + }) + + it('asks to repeat on a low-confidence transcription instead of parsing', async () => { + await verifiedUser(PHONE) + useProvider(async () => ({ text: 'withdraw 50', confidence: 0.2 })) + + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + expect(res.body).toMatch(/repeat|type your command/i) + expect(getUserForTests(PHONE)!.balance).toBe(1000) + }) + + it('responds gracefully to an unsupported audio format', async () => { + await verifiedUser(PHONE) + mockDownload.mockRejectedValue( + new UnsupportedAudioError('Unsupported audio content type: audio/x-weird') + ) + + const res = await handleWhatsAppMessage(PHONE, '', { + url: MEDIA.url, + contentType: 'audio/x-weird', + }) + + expect(res.body).toMatch(/format|type your command/i) + }) + + it('responds with a fallback when the transcription provider is down', async () => { + await verifiedUser(PHONE) + useProvider(async () => { + throw new TranscriptionUnavailableError('provider 503') + }) + + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + expect(res.body).toMatch(/aren't available right now|type your command/i) + }) + + it('falls through to unknown for transcribed nonsense (same as typed nonsense)', async () => { + await verifiedUser(PHONE) + useProvider(async () => ({ text: 'asdfghjkl qwerty', confidence: 0.95 })) + + const res = await handleWhatsAppMessage(PHONE, '', MEDIA) + + expect(res.body).toMatch(/didn't understand/i) + }) + + it('does not require confirmation for a typed (non-voice) withdraw', async () => { + await verifiedUser(PHONE, 500) + + const res = await handleWhatsAppMessage(PHONE, 'withdraw 50') + + expect(res.body).toMatch(/withdrawal request received/i) + expect(getUserForTests(PHONE)!.balance).toBe(450) + }) +}) diff --git a/tests/validation-public-routes.test.ts b/tests/validation-public-routes.test.ts index 2d73ab9..e3e5bc2 100644 --- a/tests/validation-public-routes.test.ts +++ b/tests/validation-public-routes.test.ts @@ -70,12 +70,24 @@ describe('Zod validation on public routes', () => { expect(res.body.error).toBeDefined() }) - it('returns 400 when Body is missing', async () => { + it('returns 400 when Body is missing and there is no media', async () => { const res = await request(app) .post('/api/v1/whatsapp/webhook') .send({ From: '+1234567890' }) expect(res.status).toBe(400) expect(res.body.error).toBeDefined() }) + + // A voice note (#288) has no Body, so it must clear validation and reach + // the Twilio signature check — 403 here means it passed the schema. + it('passes validation for a voice note with media and no Body', async () => { + const res = await request(app).post('/api/v1/whatsapp/webhook').send({ + From: '+1234567890', + NumMedia: '1', + MediaUrl0: 'https://api.twilio.com/media/ME123', + MediaContentType0: 'audio/ogg', + }) + expect(res.status).toBe(403) + }) }) })