diff --git a/ECOSYSTEM.md b/ECOSYSTEM.md index 25ded979..6b34f84b 100644 --- a/ECOSYSTEM.md +++ b/ECOSYSTEM.md @@ -307,18 +307,28 @@ All three pillars of the Rexone platform are fully aligned at **100% feature par ### 2. Real-Time WebSockets (Action Cable / Solid Cable) - **Endpoint**: `/cable` -- **Authentication**: JWT token sent during connection initialization or channel subscription params. -- **Channel**: `NotificationChannel` (`notification_user_{user_id}`). -- **Standard Broadcast Events**: - - `ai_response_ready`: `{ "type": "ai_response_ready", "room_id": "UUID", "message_id": "UUID" }` - - `ai_response_failed`: `{ "type": "ai_response_failed", "room_id": "UUID", "error": "Message" }` - - `tts_ready`: `{ "type": "tts_ready", "message_id": "UUID", "asset_id": "UUID" }` - - `tts_failed`: `{ "type": "tts_failed", "message_id": "UUID", "error": "Message" }` - - `asset_updated`: `{ "type": "asset_updated", "id": "UUID", "status": "optimal" | "ready" | "processing", "size_bytes": 12345, "compressed_size_bytes": 12000, "compression_ratio": "2.8%", "compression_passes": 1 }` - - `payment_success`: `{ "type": "payment_success", "product_name": "Pro Plan", "amount": "$10.00" }` - - `subscription_created` / `subscription_canceled` / `subscription_resumed`: `{ "type": "subscription_canceled", "product_name": "...", "active_until": "ISO8601" }` - - `in_app_notification`: `{ "id": "UUID", "title": "...", "message": "...", "link": "/dashboard", "read_at": null, "created_at": "ISO8601", "data": { ... } }` - - `welcome`: Sent upon first successful Action Cable subscription. +- **Authentication**: JWT token sent during connection initialization (`?token=`) or channel subscription params. +- **Channels**: + - **`NotificationChannel`** (`notification_user_{user_id}`): + - **Standard Broadcast Events**: + - `ai_response_ready`: `{ "type": "ai_response_ready", "room_id": "UUID", "message_id": "UUID" }` + - `ai_response_failed`: `{ "type": "ai_response_failed", "room_id": "UUID", "error": "Message" }` + - `tts_ready`: `{ "type": "tts_ready", "message_id": "UUID", "asset_id": "UUID" }` + - `tts_failed`: `{ "type": "tts_failed", "message_id": "UUID", "error": "Message" }` + - `asset_updated`: `{ "type": "asset_updated", "id": "UUID", "status": "optimal" | "ready" | "processing", "size_bytes": 12345, "compressed_size_bytes": 12000, "compression_ratio": "2.8%", "compression_passes": 1 }` + - `payment_success`: `{ "type": "payment_success", "product_name": "Pro Plan", "amount": "$10.00" }` + - `subscription_created` / `subscription_canceled` / `subscription_resumed`: `{ "type": "subscription_canceled", "product_name": "...", "active_until": "ISO8601" }` + - `in_app_notification`: `{ "id": "UUID", "title": "...", "message": "...", "link": "/dashboard", "read_at": null, "created_at": "ISO8601", "data": { ... } }` + - `welcome`: Sent upon first successful Action Cable subscription. + - **`SpeechLiveChannel`** (`speech_live_{user_id}`): + - **Subscription Parameters**: `{ "channel": "SpeechLiveChannel", "language": "en-US" }` + - **Client Actions (RPC)**: + - `audio`: Stream base64-encoded PCM 16-bit 16kHz mono audio chunk: `{ "action": "audio", "chunk": "" }` + - `stop`: Conclude speech recognition stream and request final transcript: `{ "action": "stop" }` + - **Server Broadcast Events**: + - `partial`: Interim transcription hypothesis `{ "type": "partial", "text": "interim text", "is_final": false }` + - `final`: Final transcription chunk `{ "type": "final", "text": "finalized sentence", "is_final": true }` + - `error`: Streaming speech recognition failure `{ "type": "error", "error": "Reason" }` ### 3. Client Telemetry Contract (`POST /v1/log/clients`) diff --git a/README.md b/README.md index 1d81f924..6204907e 100644 --- a/README.md +++ b/README.md @@ -224,9 +224,15 @@ The client provides comprehensive media upload, presentation, and compression ma The speech integration provides audio playback and streaming communication with Rexone Core's speech engine: +- **Live Audio Streaming & Recognition (`SpeechLiveChannel`)**: + - Direct microphone streaming to Rexone Core using Action Cable over WebSockets (`SpeechLiveChannel`). + - High-performance audio ingestion via `AudioWorklet` (`pcm-processor.js`) with automatic fallback to `ScriptProcessorNode`. + - Client-side downsampling from native sample rates (e.g. 44.1kHz / 48kHz) down to 16kHz mono linear PCM 16-bit, buffered into 3,200-byte chunks and base64-framed for low-latency transmission. + - Real-time RMS voice energy tracking calculating normalized voice amplitude levels (`voiceLevel: 0..1`) for visual waveform frequency bars. + - Stream lifecycle management with visibility pause/teardown handling (`document.visibilityState === "hidden"`), network disconnect recovery, and graceful session abort. - **Binary MP3 Audio Streaming**: Direct playback of synthesized audio from `POST /v1/speech/tts` returning raw binary MP3 streams without base64 wrapper overhead. -- **Chat TTS Synthesis**: Asynchronous text-to-speech generation for conversational messages, receiving `tts_ready` notifications via ActionCable and playing attached audio assets. -- **Live Audio Streaming**: Infrastructure ready for WebSocket-based live audio capture and streaming transcription (`SpeechLiveChannel`). +- **Chat TTS Synthesis**: Asynchronous text-to-speech generation for conversational messages via `SpeechController.queueTextToSpeech()`, receiving `tts_ready` notifications via Action Cable and playing attached audio assets. +- **Microphone Permissions & Web Recovery**: Browser permission guidance with lock-icon address bar instructions and instant retry flow. ### AI capabilities diff --git a/e2e/pages/sign-in-password.page.ts b/e2e/pages/sign-in-password.page.ts index 4edbf341..ff89fdfb 100644 --- a/e2e/pages/sign-in-password.page.ts +++ b/e2e/pages/sign-in-password.page.ts @@ -1,4 +1,4 @@ -import { type Locator, type Page } from "@playwright/test"; +import { expect, type Locator, type Page } from "@playwright/test"; export class SignInPasswordPage { readonly page: Page; @@ -26,6 +26,18 @@ export class SignInPasswordPage { await this.waitForVisible(); const firstInput = this.inputs.first(); await firstInput.waitFor({ state: "visible", timeout: 10000 }); + + const isLocked = await this.page + .getByText(/Too many attempts|Try again in/i) + .first() + .isVisible() + .catch(() => false); + if (isLocked) { + return; + } + + await expect(firstInput).toBeEnabled({ timeout: 10000 }); + for (let i = 0; i < 6; i++) { const input = this.inputs.nth(i); if (await input.isVisible()) { diff --git a/e2e/specs/auth/sign-in.spec.ts b/e2e/specs/auth/sign-in.spec.ts index 51fcb288..ca5a2a18 100644 --- a/e2e/specs/auth/sign-in.spec.ts +++ b/e2e/specs/auth/sign-in.spec.ts @@ -60,15 +60,37 @@ test.describe("Authentication > Sign in", () => { await authPage.submit(); await signInPage.waitForVisible(); - + // Enter wrong password up to 4 times to trigger lockout for (let i = 0; i < 4; i++) { - if (await page.getByText(/Too many attempts|Try again in/i).first().isVisible()) break; + const isLocked = await page + .getByText(/Too many attempts|Try again in/i) + .first() + .isVisible(); + if (isLocked) break; + await signInPage.enterPassword("000000"); - await page.waitForTimeout(1000); + + await Promise.race([ + page + .getByText(/Too many attempts|Try again in/i) + .first() + .waitFor({ state: "visible", timeout: 6000 }) + .catch(() => null), + page + .getByText( + /Incorrect password|Incorrect passcode|attempts remaining/i, + ) + .first() + .waitFor({ state: "visible", timeout: 6000 }) + .catch(() => null), + ]); + await page.waitForTimeout(300); } - await expect(page.getByText(/Too many attempts|Try again in/i).first()).toBeVisible(); + await expect( + page.getByText(/Too many attempts|Try again in/i).first(), + ).toBeVisible({ timeout: 10000 }); await expect(signInPage.submitButton).toBeDisabled(); }); diff --git a/e2e/specs/auth/sign-out.spec.ts b/e2e/specs/auth/sign-out.spec.ts index a47fd632..230fb059 100644 --- a/e2e/specs/auth/sign-out.spec.ts +++ b/e2e/specs/auth/sign-out.spec.ts @@ -29,7 +29,7 @@ test.describe("Authentication > Sign out", () => { await page.goto(AppRoutes.client.protected.SIGN_OUT); // Verify we are back to root/unauthenticated - await page.waitForURL(`**${AppRoutes.client.public.ROOT}`, { timeout: 10000 }); + await page.waitForURL(`**${AppRoutes.client.public.ROOT}*`, { timeout: 10000 }); await expect(page.getByRole("button", { name: /^(Enter|Sign In)$/i }).first()).toBeVisible(); }); }); diff --git a/src/AppRoutes.ts b/src/AppRoutes.ts index 3d22e20d..e17b9322 100644 --- a/src/AppRoutes.ts +++ b/src/AppRoutes.ts @@ -147,6 +147,10 @@ class AppRoutes { AI_TRANSLATE: AppRoutes.api("/ai/translate"), // POST AI_ANALYZE: AppRoutes.api("/ai/analyze"), // POST + // Speech + SPEECH_TTS: AppRoutes.api("/speech/tts"), // POST + SPEECH_STT: AppRoutes.api("/speech/stt"), // POST + // Feedback FEEDBACKS: AppRoutes.api("/feedbacks"), // GET diff --git a/src/contexts/ToastContext.tsx b/src/contexts/ToastContext.tsx index 3337b26e..f3e33125 100644 --- a/src/contexts/ToastContext.tsx +++ b/src/contexts/ToastContext.tsx @@ -8,10 +8,10 @@ import React, { } from "react"; import { createPortal } from "react-dom"; import { translate } from "../locales"; -import { Toast, TToastType } from "../design/components/overlay/Toast"; +import { Toast, type TToastTypes } from "../design/components/overlay/Toast"; interface IToastContextType { - showToast: (type: TToastType, message: string, title?: string) => void; + showToast: (type: TToastTypes, message: string, title?: string) => void; success: (message: string, title?: string) => void; error: (message: string, title?: string) => void; info: (message: string, title?: string) => void; @@ -24,13 +24,13 @@ export const ToastProvider: React.FC<{ children: ReactNode }> = ({ children, }) => { const [toast, setToast] = useState<{ - type: TToastType; + type: TToastTypes; message: string; title?: string; } | null>(null); const showToast = useCallback( - (type: TToastType, message: string, title?: string) => { + (type: TToastTypes, message: string, title?: string) => { setToast({ type, message: translate(message), title }); }, [], diff --git a/src/design/components/overlay/Toast.tsx b/src/design/components/overlay/Toast.tsx index c90d1d3e..d3687265 100644 --- a/src/design/components/overlay/Toast.tsx +++ b/src/design/components/overlay/Toast.tsx @@ -6,14 +6,14 @@ import React, { useEffect } from "react"; import { iconsLib } from "../../../assets"; import { cn } from "../../helpers"; -import { ToastTypes, type TToastType } from "../../constants"; +import { ToastTypes, type TToastTypes } from "../../constants"; export { ToastTypes }; -export type { TToastType }; +export type { TToastTypes }; export interface IToastProps { message: string; - type?: TToastType; + type?: TToastTypes; duration?: number; onClose: () => void; icon?: React.ReactNode; @@ -35,14 +35,14 @@ export const Toast: React.FC = ({ } }, [duration, onClose]); - const typeClasses = { + const typeClasses: Record = { success: "alert-success", info: "alert-info", warning: "alert-warning", error: "alert-error", }; - const defaultIcons = { + const defaultIcons: Record = { success: ( ), @@ -57,7 +57,7 @@ export const Toast: React.FC = ({ ), }; - const typeColors = { + const typeColors: Record = { success: "text-emerald-800 dark:text-emerald-200", info: "text-sky-800 dark:text-sky-200", warning: "text-amber-800 dark:text-amber-200", diff --git a/src/design/constants/toast.constants.ts b/src/design/constants/toast.constants.ts index 1ca0624d..da1664aa 100644 --- a/src/design/constants/toast.constants.ts +++ b/src/design/constants/toast.constants.ts @@ -11,4 +11,4 @@ export const ToastTypes = { ERROR: "error", } as const; -export type TToastType = (typeof ToastTypes)[keyof typeof ToastTypes]; +export type TToastTypes = (typeof ToastTypes)[keyof typeof ToastTypes]; diff --git a/src/helpers/index.ts b/src/helpers/index.ts index 6afae0a9..7a134919 100644 --- a/src/helpers/index.ts +++ b/src/helpers/index.ts @@ -1,3 +1,4 @@ export * from "./jwt.helper"; export * from "./date.helper"; +export * from "./socket.helpers.ts"; export * from "../design/helpers"; diff --git a/src/helpers/socket.helpers.ts b/src/helpers/socket.helpers.ts new file mode 100644 index 00000000..4dc3efed --- /dev/null +++ b/src/helpers/socket.helpers.ts @@ -0,0 +1,120 @@ +import { ToastTypes, type TToastTypes } from "../design"; +import { NOTIFICATION_SOCKET_TYPES } from "../modules/notification"; + +export type ISocketMessage = { + type: string; + message?: string; + data?: Record; + created_at?: string; + channel?: string; +}; + +export const SOCKET_MESSAGE_TYPES = { + NOTIFICATION: "notification", + SPEECH: "speech", +} as const; + +export const SOCKET_CHANNELS = { + NOTIFICATION: "NotificationChannel", + SPEECH_LIVE: "SpeechLiveChannel", +} as const; + +export const SPEECH_EVENT_TYPES = { + PARTIAL: "partial", + FINAL: "final", + ERROR: "error", +} as const; + +export const SOCKET_SUBSCRIBE_TIMEOUT_MS = 10_000; +export const SOCKET_CONNECT_WAIT_MS = 5_000; + +export const NOTIFICATION_TOAST_SUCCESS_TYPES = [ + NOTIFICATION_SOCKET_TYPES.PAYMENT_SUCCESS, + NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CREATED, + NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_RESUMED, + NOTIFICATION_SOCKET_TYPES.WELCOME, + NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_READY, + NOTIFICATION_SOCKET_TYPES.TTS_READY, + NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSED, +] as const; + +export const NOTIFICATION_TOAST_ERROR_TYPES = [ + NOTIFICATION_SOCKET_TYPES.PAYMENT_FAILED, + NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CANCELED, + NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_FAILED, + NOTIFICATION_SOCKET_TYPES.TTS_FAILED, + NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSION_FAILED, +] as const; + +export function parseCableChannel(identifier: unknown): string { + if (identifier && typeof identifier === "object") { + const channel = (identifier as { channel?: unknown }).channel; + return typeof channel === "string" ? channel : ""; + } + + if (typeof identifier !== "string" || identifier.trim() === "") { + return ""; + } + + try { + const parsed = JSON.parse(identifier) as { channel?: unknown }; + return typeof parsed.channel === "string" ? parsed.channel : ""; + } catch { + return ""; + } +} + +export function getSpeechEventType(data: ISocketMessage): string { + return typeof data.data?.type === "string" ? data.data.type : ""; +} + +export function isSpeechLiveMessage(data: ISocketMessage): boolean { + if (data.channel === SOCKET_CHANNELS.SPEECH_LIVE) { + return true; + } + + if (data.type === SOCKET_MESSAGE_TYPES.SPEECH) { + const eventType = getSpeechEventType(data); + return ( + eventType === SPEECH_EVENT_TYPES.PARTIAL || + eventType === SPEECH_EVENT_TYPES.FINAL || + eventType === SPEECH_EVENT_TYPES.ERROR + ); + } + + return false; +} + +export function getSocketToast(data: ISocketMessage): { + kind: TToastTypes; + message: string; +} | null { + const message = data.message?.trim() ?? ""; + + if (isSpeechLiveMessage(data)) { + if (getSpeechEventType(data) === SPEECH_EVENT_TYPES.ERROR && message) { + return { kind: ToastTypes.ERROR, message }; + } + return null; + } + + if (data.type !== SOCKET_MESSAGE_TYPES.NOTIFICATION || !message) { + return null; + } + + const eventType = getSpeechEventType(data); + + if ( + (NOTIFICATION_TOAST_SUCCESS_TYPES as readonly string[]).includes(eventType) + ) { + return { kind: ToastTypes.SUCCESS, message }; + } + + if ( + (NOTIFICATION_TOAST_ERROR_TYPES as readonly string[]).includes(eventType) + ) { + return { kind: ToastTypes.ERROR, message }; + } + + return { kind: ToastTypes.INFO, message }; +} diff --git a/src/hooks/useSocket.ts b/src/hooks/useSocket.ts index 8463448c..de84ae97 100644 --- a/src/hooks/useSocket.ts +++ b/src/hooks/useSocket.ts @@ -1,8 +1,9 @@ import { useEffect, useState, useCallback } from "react"; import { useAuth } from "../contexts/AuthContext"; import { useToast } from "../contexts/ToastContext"; +import { getSocketToast, SOCKET_MESSAGE_TYPES } from "../helpers/socket.helpers"; import SocketService, { ISocketMessage } from "../services/socket.service"; -import { NOTIFICATION_SOCKET_TYPES } from "../modules/notification/constants"; +import { ToastTypes } from "../constants"; export interface INotification { id: string; @@ -24,11 +25,13 @@ export const useSocket = () => { } const handleNotification = (data: ISocketMessage) => { - if (data.type === "notification") { - if (!data.message) { - return; - } + const toast = getSocketToast(data); + if (!toast) { + return; + } + // Only accumulate in persistent notification state if payload is a domain notification + if (data.type === SOCKET_MESSAGE_TYPES.NOTIFICATION && data.message) { const notif: INotification = { id: Date.now().toString() + Math.random().toString(36).substr(2, 5), message: data.message, @@ -36,37 +39,22 @@ export const useSocket = () => { created_at: data.created_at || new Date().toISOString(), }; setNotifications((prev) => { - // Prevent duplicates if (prev.some((n) => n.id === notif.id)) return prev; return [notif, ...prev]; }); + } - const type = typeof notif.data.type === "string" ? notif.data.type : "general"; + if (toast.kind === ToastTypes.SUCCESS) { + success(toast.message); + return; + } - switch (type) { - case NOTIFICATION_SOCKET_TYPES.PAYMENT_SUCCESS: - case NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CREATED: - case NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_RESUMED: - case NOTIFICATION_SOCKET_TYPES.WELCOME: - case NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_READY: - case NOTIFICATION_SOCKET_TYPES.TTS_READY: - case NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSED: - success(notif.message); - break; - case NOTIFICATION_SOCKET_TYPES.PAYMENT_FAILED: - case NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CANCELED: - case NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_FAILED: - case NOTIFICATION_SOCKET_TYPES.TTS_FAILED: - case NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSION_FAILED: - error(notif.message); - break; - case NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSING: - case NOTIFICATION_SOCKET_TYPES.SIGN_IN_ALERT: - default: - info(notif.message); - break; - } + if (toast.kind === ToastTypes.ERROR) { + error(toast.message); + return; } + + info(toast.message); }; SocketService.addListener(handleNotification); diff --git a/src/locales/app_locales.ts b/src/locales/app_locales.ts index 7e4e712d..864ae4ba 100644 --- a/src/locales/app_locales.ts +++ b/src/locales/app_locales.ts @@ -998,6 +998,7 @@ export const AppLocales = { Success: "common.success", NotFound: "common.not_found", NotFoundDesc: "common.not_found_desc", + Retry: "common.retry", }, User: { Profile: "user.profile", @@ -1023,10 +1024,24 @@ export const AppLocales = { Title: "ai.title", Rooms: "ai.rooms", NewChat: "ai.new_chat", + DefaultGreeting: "ai.default_greeting", TypeMessage: "ai.type_message", Send: "ai.send", Processing: "ai.processing", ClearHistory: "ai.clear_history", + ClearHistoryConfirm: "ai.clear_history_confirm", + Listen: "ai.listen", + Thinking: "ai.thinking", + CancelListening: "ai.cancel_listening", + MicPermissionTitle: "ai.mic_permission_title", + MicPermissionMessage: "ai.mic_permission_message", + OpenSettings: "ai.open_settings", + Retry: "ai.retry", + StartRecordingFailed: "ai.ai_start_recording_failed", + TranscriptionFailed: "ai.ai_transcription_failed", + SendMessageFailed: "ai.ai_send_message_failed", + TtsFailed: "ai.ai_tts_failed", + TtsEmpty: "ai.ai_tts_empty", Errors: { Unexpected: "ai.errors.unexpected", LoadRooms: "ai.errors.load_rooms", @@ -1038,6 +1053,13 @@ export const AppLocales = { DeleteRoom: "ai.errors.delete_room", }, }, + Speech: { + TtsFailed: "speech.tts_failed", + SttFailed: "speech.stt_failed", + PlaybackFailed: "speech.playback_failed", + PermissionDenied: "speech.permission_denied", + AudioRequired: "speech.audio_required", + }, Payment: { Title: "payment.title", Subscriptions: "payment.subscriptions", diff --git a/src/locales/en.json b/src/locales/en.json index 704da3fd..eb64eaf1 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -11,7 +11,8 @@ "error": "Error", "success": "Success", "not_found": "Page Not Found", - "not_found_desc": "The page you are looking for does not exist." + "not_found_desc": "The page you are looking for does not exist.", + "retry": "Retry" }, "user": { @@ -51,6 +52,20 @@ "send": "Send", "processing": "AI is thinking...", "clear_history": "Clear History", + "clear_history_confirm": "All messages in this conversation will be permanently deleted. Are you sure you want to clear chat history?", + "default_greeting": "Hello! I'm your AI assistant. How can I help you today?", + "listen": "Listen", + "thinking": "AI is thinking", + "cancel_listening": "Cancel listening", + "mic_permission_title": "Microphone access required", + "mic_permission_message": "Voice input needs microphone access. Please allow microphone access in your browser address bar (lock icon) to use voice input.", + "open_settings": "Open Settings", + "retry": "Retry", + "ai_start_recording_failed": "Failed to start recording", + "ai_transcription_failed": "Failed to transcribe audio", + "ai_send_message_failed": "Failed to send message", + "ai_tts_failed": "Failed to play speech", + "ai_tts_empty": "Nothing to speak", "errors": { "unexpected": "An error occurred. Please try again.", "load_rooms": "Failed to load rooms", @@ -63,6 +78,14 @@ } }, + "speech": { + "tts_failed": "Failed to synthesize speech", + "stt_failed": "Failed to transcribe audio", + "playback_failed": "Audio playback failed", + "permission_denied": "Microphone permission was denied", + "audio_required": "Audio file or URL is required" + }, + "payment": { "title": "Billing & Subscriptions", "subscriptions": "Subscriptions", diff --git a/src/locales/my.json b/src/locales/my.json index 7c04e49a..10017471 100644 --- a/src/locales/my.json +++ b/src/locales/my.json @@ -11,7 +11,8 @@ "error": "အမှား", "success": "အောင်မြင်သည်", "not_found": "ရှာမတွေ့ပါ", - "not_found_desc": "ရှာဖွေနေသော စာမျက်နှာ မရှိပါ။" + "not_found_desc": "ရှာဖွေနေသော စာမျက်နှာ မရှိပါ။", + "retry": "ထပ်စမ်းမည်" }, "user": { @@ -51,6 +52,20 @@ "send": "ပို့မည်", "processing": "AI စဉ်းစားနေဆဲ...", "clear_history": "မှတ်တမ်းရှင်းမည်", + "clear_history_confirm": "ဤစကားဝိုင်းရှိ မက်ဆေ့ဂျ်အားလုံးကို အပြီးအပိုင် ဖျက်မည်။ မှတ်တမ်းကို ရှင်းလင်းရန် သေချာပါသလား။", + "default_greeting": "မင်္ဂလာပါ! ကျွန်တော်သည် သင်၏ AI လက်ထောက် ဖြစ်ပါသည်။ ဘာများ ကူညီပေးရမလဲ?", + "listen": "နားထောင်မည်", + "thinking": "AI စဉ်းစားနေသည်", + "cancel_listening": "နားထောင်ခြင်း ရပ်တန့်မည်", + "mic_permission_title": "မိုက်ခရိုဖုန်း ခွင့်ပြုချက် လိုအပ်သည်", + "mic_permission_message": "အသံဖြင့် ရိုက်ထည့်ရန် မိုက်ခရိုဖုန်း ခွင့်ပြုချက် လိုအပ်ပါသည်။ အသံသွင်းရန်အတွက် browser လိပ်စာဘားရှိ သော့ခတ်ပုံ (lock icon) မှ မိုက်ခရိုဖုန်းကို ခွင့်ပြုပေးပါ။", + "open_settings": "Settings ဖွင့်ရန်", + "retry": "ထပ်စမ်းမည်", + "ai_start_recording_failed": "အသံဖမ်းခြင်း စတင်၍ မရပါ", + "ai_transcription_failed": "အသံကို စာသားမပြောင်း၍ ရပါ", + "ai_send_message_failed": "မက်ဆေ့ဂျ် ပို့၍ မရပါ", + "ai_tts_failed": "အသံဖွင့်၍ မရပါ", + "ai_tts_empty": "ဖွင့်ရန် စာသားမရှိပါ", "errors": { "unexpected": "အမှားဖြစ်သွားပါသည်။ ထပ်စမ်းပါ။", "load_rooms": "အခန်းများ မရပါ", @@ -63,6 +78,14 @@ } }, + "speech": { + "tts_failed": "အသံပြောင်းလဲခြင်း မအောင်မြင်ပါ", + "stt_failed": "အသံကို စာသားပြောင်း၍ မရပါ", + "playback_failed": "အသံဖွင့်၍ မရပါ", + "permission_denied": "မိုက်ခရိုဖုန်း ခွင့်ပြုချက် ငြင်းပယ်ခံရပါသည်", + "audio_required": "အသံဖိုင် သို့မဟုတ် လင့်ခ် လိုအပ်ပါသည်" + }, + "payment": { "title": "ငွေပေးချေမှုနှင့် စာရင်းသွင်းမှု", "subscriptions": "စာရင်းသွင်းမှုများ", diff --git a/src/modules/admin/components/AdminSidebarNav.tsx b/src/modules/admin/components/AdminSidebarNav.tsx index 48f8716d..696a56da 100644 --- a/src/modules/admin/components/AdminSidebarNav.tsx +++ b/src/modules/admin/components/AdminSidebarNav.tsx @@ -193,10 +193,7 @@ export const AdminSidebarNav: React.FC = ({ )} >
{enabledItems.map((section, idx) => { const isCollapsed = collapsedSections[section.id] ?? false; diff --git a/src/modules/ai/ai.controller.test.ts b/src/modules/ai/ai.controller.test.ts index 5d1d5b94..0d49f9cf 100644 --- a/src/modules/ai/ai.controller.test.ts +++ b/src/modules/ai/ai.controller.test.ts @@ -4,6 +4,13 @@ import AiService from "./ai.service"; import SocketService from "../../services/socket.service"; import { IRoom, IMessage } from "./types"; import { AI_MESSAGE_STATUS, AI_SOCKET_EVENTS } from "./constants"; +import { SpeechController } from "../speech"; + +vi.mock("../speech", () => ({ + SpeechController: { + queueTextToSpeech: vi.fn(), + }, +})); vi.mock("./ai.service", () => ({ default: { @@ -234,4 +241,36 @@ describe("AiController", () => { expect(AiController.getCurrentRoomId()).toBeNull(); }); }); + + describe("queueTextToSpeech", () => { + it("delegates to SpeechController and returns success", async () => { + vi.mocked(SpeechController.queueTextToSpeech).mockResolvedValue({ + success: true, + message: "TTS queued", + }); + + const result = await AiController.queueTextToSpeech("msg-1"); + + expect(SpeechController.queueTextToSpeech).toHaveBeenCalledWith("msg-1"); + expect(result).toEqual({ + success: true, + message: "TTS queued", + }); + }); + + it("delegates to SpeechController and returns failure", async () => { + vi.mocked(SpeechController.queueTextToSpeech).mockResolvedValue({ + success: false, + error: "Failed to queue TTS", + }); + + const result = await AiController.queueTextToSpeech("msg-1"); + + expect(SpeechController.queueTextToSpeech).toHaveBeenCalledWith("msg-1"); + expect(result).toEqual({ + success: false, + error: "Failed to queue TTS", + }); + }); + }); }); diff --git a/src/modules/ai/ai.controller.ts b/src/modules/ai/ai.controller.ts index 758af6d7..521db310 100644 --- a/src/modules/ai/ai.controller.ts +++ b/src/modules/ai/ai.controller.ts @@ -5,11 +5,18 @@ import { IMessage, IRoom } from "./types"; import SocketService, { ISocketMessage } from "../../services/socket.service"; import { IApiPagination } from "../../models"; import { AppLocales, translate } from "../../locales"; -import { AI_DEFAULTS, AI_MESSAGE_STATUS, AI_SOCKET_EVENTS } from "./constants"; - -const AI_RESPONSE_EVENT_TYPES: readonly string[] = [ +import { + AI_DEFAULTS, + AI_MESSAGE_STATUS, + AI_SOCKET_EVENTS, +} from "./constants"; +import { SpeechController } from "../speech"; + +const AI_SOCKET_EVENT_TYPES: readonly string[] = [ AI_SOCKET_EVENTS.RESPONSE_READY, AI_SOCKET_EVENTS.RESPONSE_FAILED, + AI_SOCKET_EVENTS.TTS_READY, + AI_SOCKET_EVENTS.TTS_FAILED, ]; const AI_PROCESSING_MESSAGE_STATUSES: readonly string[] = [ @@ -29,26 +36,41 @@ class AiController { return this.currentRoomId; } - subscribeToAiMessages(callback: () => void): () => void { + subscribeToAiMessages( + callback: (eventType: string) => void, + ): () => void { const handleAiMessage = (event: ISocketMessage) => { - const eventType = typeof event.data?.type === "string" ? event.data.type : ""; - const roomId = typeof event.data?.room_id === "string" ? event.data.room_id : ""; + const eventType = + typeof event.data?.type === "string" ? event.data.type : ""; + const roomId = + typeof event.data?.room_id === "string" ? event.data.room_id : ""; if ( event.type !== "notification" || - !AI_RESPONSE_EVENT_TYPES.includes(eventType) || - roomId !== this.currentRoomId + !AI_SOCKET_EVENT_TYPES.includes(eventType) ) { return; } - callback(); + if (roomId && this.currentRoomId && roomId !== this.currentRoomId) { + return; + } + + callback(eventType); }; SocketService.addListener(handleAiMessage); return () => SocketService.removeListener(handleAiMessage); } + async queueTextToSpeech(messageId: string): Promise<{ + success: boolean; + message?: string; + error?: string; + }> { + return SpeechController.queueTextToSpeech(messageId); + } + async getRooms(params?: { page?: number; limit?: number }): Promise<{ success: boolean; rooms: IRoom[]; diff --git a/src/modules/ai/constants.ts b/src/modules/ai/constants.ts index f0fdbb56..342b4200 100644 --- a/src/modules/ai/constants.ts +++ b/src/modules/ai/constants.ts @@ -22,8 +22,11 @@ export type TAiMessageStatus = export const AI_SOCKET_EVENTS = { RESPONSE_READY: "ai_response_ready", RESPONSE_FAILED: "ai_response_failed", + TTS_READY: "tts_ready", + TTS_FAILED: "tts_failed", } as const; + export const AI_DEFAULTS = { TEMPERATURE: 0.7, MAX_TOKENS: 2000, diff --git a/src/modules/ai/pages/AiPage.tsx b/src/modules/ai/pages/AiPage.tsx index 711407f2..6e0bceb3 100644 --- a/src/modules/ai/pages/AiPage.tsx +++ b/src/modules/ai/pages/AiPage.tsx @@ -3,18 +3,40 @@ import React, { useState, useRef, useEffect, useCallback } from "react"; import { Button, TextArea, ConfirmDialog } from "../../../design/components"; import { useToast, useLoading } from "../../../contexts"; +import { AppLocales, useTranslate } from "../../../locales"; +import { iconsLib } from "../../../assets"; import AiController from "../ai.controller"; -import { IMessage } from ".."; +import { AI_SOCKET_EVENTS, getMessageAudioUrl, IMessage } from ".."; +import { SPEECH_LISTEN_RESULTS, useLiveSpeech } from "../../speech"; import { ADMIN_CHAT_ROLES } from "../../admin"; +import { ButtonSizes, ButtonVariants } from "../../../design"; export const AiPage: React.FC = () => { - const { success, error } = useToast(); + const t = useTranslate(); + const { success, error, info } = useToast(); const { isLoading, setLoading } = useLoading(); + const { + liveText, + voiceLevel, + isListenSessionActive, + isPlaying, + startListening, + stopListening, + playUrl, + stopPlayback, + } = useLiveSpeech(); const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); const [isProcessing, setIsProcessing] = useState(false); const [isClearDialogOpen, setIsClearDialogOpen] = useState(false); + const [isMicDialogOpen, setIsMicDialogOpen] = useState(false); + const [activeTtsMessageId, setActiveTtsMessageId] = useState( + null, + ); + const [playingMessageId, setPlayingMessageId] = useState(null); + const textBeforeListenRef = useRef(""); const messagesEndRef = useRef(null); + const greeting = t(AppLocales.Ai.DefaultGreeting); const scrollToBottom = () => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); @@ -28,8 +50,8 @@ export const AiPage: React.FC = () => { setMessages([ { id: "welcome", - role: "assistant", - content: "Hello! I'm your AI assistant. How can I help you today?", + role: ADMIN_CHAT_ROLES.ASSISTANT, + content: greeting, created_at: new Date().toISOString(), }, ]); @@ -42,14 +64,14 @@ export const AiPage: React.FC = () => { setMessages([ { id: "welcome", - role: "assistant", - content: "Hello! I'm your AI assistant. How can I help you today?", + role: ADMIN_CHAT_ROLES.ASSISTANT, + content: greeting, created_at: new Date().toISOString(), }, ]); setIsProcessing(false); } - }, []); + }, [greeting]); useEffect(() => { scrollToBottom(); @@ -64,14 +86,173 @@ export const AiPage: React.FC = () => { }, [loadHistory]); useEffect(() => { - const unsubscribe = AiController.subscribeToAiMessages(() => { - loadHistory(); + const unsubscribe = AiController.subscribeToAiMessages((eventType) => { + void loadHistory(); + if ( + eventType === AI_SOCKET_EVENTS.TTS_READY || + eventType === AI_SOCKET_EVENTS.TTS_FAILED + ) { + setActiveTtsMessageId(null); + } }); return unsubscribe; }, [loadHistory]); + useEffect(() => { + if (!isPlaying) { + setPlayingMessageId(null); + } + }, [isPlaying]); + + useEffect(() => { + if (isListenSessionActive) { + setInput(liveText); + } + }, [isListenSessionActive, liveText]); + + const handleListenResult = (result: string) => { + if ( + result === SPEECH_LISTEN_RESULTS.STARTED || + result === SPEECH_LISTEN_RESULTS.ALREADY_LISTENING + ) { + return; + } + + if (result === SPEECH_LISTEN_RESULTS.PERMISSION_DENIED) { + setIsMicDialogOpen(true); + return; + } + + if (result === SPEECH_LISTEN_RESULTS.DISCONNECTED) { + error(t(AppLocales.Ai.TranscriptionFailed)); + return; + } + + error(t(AppLocales.Ai.StartRecordingFailed)); + }; + + const handleStartListening = async () => { + if (isLoading || isProcessing || isListenSessionActive) { + return; + } + + textBeforeListenRef.current = input; + const result = await startListening({ seed: input }); + handleListenResult(result); + }; + + const handleStopListening = async () => { + await stopListening(); + }; + + const handleCancelListening = async () => { + await stopListening(); + setInput(textBeforeListenRef.current); + }; + + const handleMicPermissionConfirm = async () => { + setIsMicDialogOpen(false); + const result = await startListening({ seed: textBeforeListenRef.current }); + handleListenResult(result); + }; + + const isTtsDisabledForMessage = (messageId: string) => { + if (isListenSessionActive) { + return true; + } + + return Boolean(activeTtsMessageId && activeTtsMessageId !== messageId); + }; + + const handleTtsTap = async (message: IMessage) => { + if (isTtsDisabledForMessage(message.id)) { + return; + } + + if (playingMessageId === message.id && isPlaying) { + stopPlayback(); + setPlayingMessageId(null); + return; + } + + stopPlayback(); + setPlayingMessageId(null); + + const audioUrl = getMessageAudioUrl(message); + if (audioUrl) { + try { + await playUrl(audioUrl); + setPlayingMessageId(message.id); + } catch { + error(t(AppLocales.Ai.TtsFailed)); + } + return; + } + + if (!message.content.trim()) { + error(t(AppLocales.Ai.TtsEmpty)); + return; + } + + setActiveTtsMessageId(message.id); + const result = await AiController.queueTextToSpeech(message.id); + + if (result.success) { + if (result.message) { + info(result.message); + } + return; + } + + setActiveTtsMessageId(null); + error(result.error || t(AppLocales.Ai.TtsFailed)); + }; + + const renderTtsControl = (message: IMessage) => { + if ( + message.role !== ADMIN_CHAT_ROLES.ASSISTANT || + !message.content.trim() + ) { + return null; + } + + const isQueued = activeTtsMessageId === message.id; + const isThisPlaying = playingMessageId === message.id && isPlaying; + const hasAudio = Boolean(getMessageAudioUrl(message)); + const disabled = isTtsDisabledForMessage(message.id); + + if (isQueued) { + return ( + + ); + } + + let Icon = iconsLib.speaker; + if (isThisPlaying) { + Icon = iconsLib.stop; + } else if (hasAudio) { + Icon = iconsLib.play; + } + + return ( + + ); + }; + const handleSend = async () => { - if (!input.trim() || isLoading || isProcessing) return; + if (!input.trim() || isLoading || isProcessing || isListenSessionActive) { + return; + } const content = input.trim(); setInput(""); @@ -85,22 +266,27 @@ export const AiPage: React.FC = () => { result.message!, ]); setIsProcessing(true); - success("Message sent. AI is thinking while you continue browsing."); + success(t(AppLocales.Ai.Processing)); } else { setInput(content); - error(result.error || "Failed to get AI response"); + error(result.error || t(AppLocales.Ai.SendMessageFailed)); } setLoading(false); }; const handleKeyDown = (e: React.KeyboardEvent) => { + if (isListenSessionActive) { + e.preventDefault(); + return; + } + if ( (e.key === "Enter" && !e.shiftKey) || ((e.ctrlKey || e.metaKey) && e.key === "Enter") ) { e.preventDefault(); - handleSend(); + void handleSend(); } }; @@ -109,46 +295,43 @@ export const AiPage: React.FC = () => { const result = await AiController.clearHistory(null); if (result.success) { - success("Chat history cleared"); + success(t(AppLocales.Ai.ClearHistory)); setMessages([ { id: "welcome", - role: "assistant", - content: "Hello! I'm your AI assistant. How can I help you today?", + role: ADMIN_CHAT_ROLES.ASSISTANT, + content: greeting, created_at: new Date().toISOString(), }, ]); } else { - error(result.error || "Failed to clear history"); + error(result.error || t(AppLocales.Ai.Errors.ClearHistory)); } }; + const canListen = !isLoading && !isProcessing && !isListenSessionActive; + return ( <>
- {/* Header */}
-

🤖 AI Assistant

-

- Powered by DeepSeek AI -

+

{t(AppLocales.Ai.Title)}

{messages.length > 1 && ( )}
- {/* Messages */}
{messages.map((message) => (
{ {message.metadata?.status === "failed" && ( - AI could not complete this message. You can try again. + {t(AppLocales.Ai.Errors.GetResponse)} )} + {renderTtsControl(message)}
))} @@ -184,7 +368,7 @@ export const AiPage: React.FC = () => { {isProcessing && (
-

AI is thinking…

+

{t(AppLocales.Ai.Thinking)}

@@ -193,7 +377,6 @@ export const AiPage: React.FC = () => {
- {/* Input */}