Feat/ps speech service

ECOSYSTEM.md

@@ -307,18 +307,28 @@ All three pillars of the Rexone platform are fully aligned at **100% feature par
 ### 2. Real-Time WebSockets (Action Cable / Solid Cable)
 
 - **Endpoint**: `/cable`
-- **Authentication**: JWT token sent during connection initialization or channel subscription params.
-- **Channel**: `NotificationChannel` (`notification_user_{user_id}`).
-- **Standard Broadcast Events**:
-  - `ai_response_ready`: `{ "type": "ai_response_ready", "room_id": "UUID", "message_id": "UUID" }`
-  - `ai_response_failed`: `{ "type": "ai_response_failed", "room_id": "UUID", "error": "Message" }`
-  - `tts_ready`: `{ "type": "tts_ready", "message_id": "UUID", "asset_id": "UUID" }`
-  - `tts_failed`: `{ "type": "tts_failed", "message_id": "UUID", "error": "Message" }`
-  - `asset_updated`: `{ "type": "asset_updated", "id": "UUID", "status": "optimal" | "ready" | "processing", "size_bytes": 12345, "compressed_size_bytes": 12000, "compression_ratio": "2.8%", "compression_passes": 1 }`
-  - `payment_success`: `{ "type": "payment_success", "product_name": "Pro Plan", "amount": "$10.00" }`
-  - `subscription_created` / `subscription_canceled` / `subscription_resumed`: `{ "type": "subscription_canceled", "product_name": "...", "active_until": "ISO8601" }`
-  - `in_app_notification`: `{ "id": "UUID", "title": "...", "message": "...", "link": "/dashboard", "read_at": null, "created_at": "ISO8601", "data": { ... } }`
-  - `welcome`: Sent upon first successful Action Cable subscription.
+- **Authentication**: JWT token sent during connection initialization (`?token=<JWT>`) or channel subscription params.
+- **Channels**:
+  - **`NotificationChannel`** (`notification_user_{user_id}`):
+    - **Standard Broadcast Events**:
+      - `ai_response_ready`: `{ "type": "ai_response_ready", "room_id": "UUID", "message_id": "UUID" }`
+      - `ai_response_failed`: `{ "type": "ai_response_failed", "room_id": "UUID", "error": "Message" }`
+      - `tts_ready`: `{ "type": "tts_ready", "message_id": "UUID", "asset_id": "UUID" }`
+      - `tts_failed`: `{ "type": "tts_failed", "message_id": "UUID", "error": "Message" }`
+      - `asset_updated`: `{ "type": "asset_updated", "id": "UUID", "status": "optimal" | "ready" | "processing", "size_bytes": 12345, "compressed_size_bytes": 12000, "compression_ratio": "2.8%", "compression_passes": 1 }`
+      - `payment_success`: `{ "type": "payment_success", "product_name": "Pro Plan", "amount": "$10.00" }`
+      - `subscription_created` / `subscription_canceled` / `subscription_resumed`: `{ "type": "subscription_canceled", "product_name": "...", "active_until": "ISO8601" }`
+      - `in_app_notification`: `{ "id": "UUID", "title": "...", "message": "...", "link": "/dashboard", "read_at": null, "created_at": "ISO8601", "data": { ... } }`
+      - `welcome`: Sent upon first successful Action Cable subscription.
+  - **`SpeechLiveChannel`** (`speech_live_{user_id}`):
+    - **Subscription Parameters**: `{ "channel": "SpeechLiveChannel", "language": "en-US" }`
+    - **Client Actions (RPC)**:
+      - `audio`: Stream base64-encoded PCM 16-bit 16kHz mono audio chunk: `{ "action": "audio", "chunk": "<base64_pcm>" }`
+      - `stop`: Conclude speech recognition stream and request final transcript: `{ "action": "stop" }`
+    - **Server Broadcast Events**:
+      - `partial`: Interim transcription hypothesis `{ "type": "partial", "text": "interim text", "is_final": false }`
+      - `final`: Final transcription chunk `{ "type": "final", "text": "finalized sentence", "is_final": true }`
+      - `error`: Streaming speech recognition failure `{ "type": "error", "error": "Reason" }`
 
 ### 3. Client Telemetry Contract (`POST /v1/log/clients`)
 

README.md

@@ -224,9 +224,15 @@ The client provides comprehensive media upload, presentation, and compression ma
 
 The speech integration provides audio playback and streaming communication with Rexone Core's speech engine:
 
+- **Live Audio Streaming & Recognition (`SpeechLiveChannel`)**:
+  - Direct microphone streaming to Rexone Core using Action Cable over WebSockets (`SpeechLiveChannel`).
+  - High-performance audio ingestion via `AudioWorklet` (`pcm-processor.js`) with automatic fallback to `ScriptProcessorNode`.
+  - Client-side downsampling from native sample rates (e.g. 44.1kHz / 48kHz) down to 16kHz mono linear PCM 16-bit, buffered into 3,200-byte chunks and base64-framed for low-latency transmission.
+  - Real-time RMS voice energy tracking calculating normalized voice amplitude levels (`voiceLevel: 0..1`) for visual waveform frequency bars.
+  - Stream lifecycle management with visibility pause/teardown handling (`document.visibilityState === "hidden"`), network disconnect recovery, and graceful session abort.
 - **Binary MP3 Audio Streaming**: Direct playback of synthesized audio from `POST /v1/speech/tts` returning raw binary MP3 streams without base64 wrapper overhead.
-- **Chat TTS Synthesis**: Asynchronous text-to-speech generation for conversational messages, receiving `tts_ready` notifications via ActionCable and playing attached audio assets.
-- **Live Audio Streaming**: Infrastructure ready for WebSocket-based live audio capture and streaming transcription (`SpeechLiveChannel`).
+- **Chat TTS Synthesis**: Asynchronous text-to-speech generation for conversational messages via `SpeechController.queueTextToSpeech()`, receiving `tts_ready` notifications via Action Cable and playing attached audio assets.
+- **Microphone Permissions & Web Recovery**: Browser permission guidance with lock-icon address bar instructions and instant retry flow.
 
 ### AI capabilities
 

e2e/pages/sign-in-password.page.ts

@@ -1,4 +1,4 @@
-import { type Locator, type Page } from "@playwright/test";
+import { expect, type Locator, type Page } from "@playwright/test";
 
 export class SignInPasswordPage {
   readonly page: Page;
@@ -26,6 +26,18 @@ export class SignInPasswordPage {
     await this.waitForVisible();
     const firstInput = this.inputs.first();
     await firstInput.waitFor({ state: "visible", timeout: 10000 });
+
+    const isLocked = await this.page
+      .getByText(/Too many attempts|Try again in/i)
+      .first()
+      .isVisible()
+      .catch(() => false);
+    if (isLocked) {
+      return;
+    }
+
+    await expect(firstInput).toBeEnabled({ timeout: 10000 });
+
     for (let i = 0; i < 6; i++) {
       const input = this.inputs.nth(i);
       if (await input.isVisible()) {

e2e/specs/auth/sign-in.spec.ts

@@ -60,15 +60,37 @@ test.describe("Authentication > Sign in", () => {
     await authPage.submit();
 
     await signInPage.waitForVisible();
-    
+
     // Enter wrong password up to 4 times to trigger lockout
     for (let i = 0; i < 4; i++) {
-      if (await page.getByText(/Too many attempts|Try again in/i).first().isVisible()) break;
+      const isLocked = await page
+        .getByText(/Too many attempts|Try again in/i)
+        .first()
+        .isVisible();
+      if (isLocked) break;
+
       await signInPage.enterPassword("000000");
-      await page.waitForTimeout(1000);
+
+      await Promise.race([
+        page
+          .getByText(/Too many attempts|Try again in/i)
+          .first()
+          .waitFor({ state: "visible", timeout: 6000 })
+          .catch(() => null),
+        page
+          .getByText(
+            /Incorrect password|Incorrect passcode|attempts remaining/i,
+          )
+          .first()
+          .waitFor({ state: "visible", timeout: 6000 })
+          .catch(() => null),
+      ]);
+      await page.waitForTimeout(300);
     }
 
-    await expect(page.getByText(/Too many attempts|Try again in/i).first()).toBeVisible();
+    await expect(
+      page.getByText(/Too many attempts|Try again in/i).first(),
+    ).toBeVisible({ timeout: 10000 });
     await expect(signInPage.submitButton).toBeDisabled();
   });
 

e2e/specs/auth/sign-out.spec.ts

@@ -29,7 +29,7 @@ test.describe("Authentication > Sign out", () => {
     await page.goto(AppRoutes.client.protected.SIGN_OUT);
 
     // Verify we are back to root/unauthenticated
-    await page.waitForURL(`**${AppRoutes.client.public.ROOT}`, { timeout: 10000 });
+    await page.waitForURL(`**${AppRoutes.client.public.ROOT}*`, { timeout: 10000 });
     await expect(page.getByRole("button", { name: /^(Enter|Sign In)$/i }).first()).toBeVisible();
   });
 });

src/AppRoutes.ts

@@ -147,6 +147,10 @@ class AppRoutes {
       AI_TRANSLATE: AppRoutes.api("/ai/translate"), // POST
       AI_ANALYZE: AppRoutes.api("/ai/analyze"), // POST
 
+      // Speech
+      SPEECH_TTS: AppRoutes.api("/speech/tts"), // POST
+      SPEECH_STT: AppRoutes.api("/speech/stt"), // POST
+
       // Feedback
       FEEDBACKS: AppRoutes.api("/feedbacks"), // GET
 

src/contexts/ToastContext.tsx

@@ -8,10 +8,10 @@ import React, {
 } from "react";
 import { createPortal } from "react-dom";
 import { translate } from "../locales";
-import { Toast, TToastType } from "../design/components/overlay/Toast";
+import { Toast, type TToastTypes } from "../design/components/overlay/Toast";
 
 interface IToastContextType {
-  showToast: (type: TToastType, message: string, title?: string) => void;
+  showToast: (type: TToastTypes, message: string, title?: string) => void;
   success: (message: string, title?: string) => void;
   error: (message: string, title?: string) => void;
   info: (message: string, title?: string) => void;
@@ -24,13 +24,13 @@ export const ToastProvider: React.FC<{ children: ReactNode }> = ({
   children,
 }) => {
   const [toast, setToast] = useState<{
-    type: TToastType;
+    type: TToastTypes;
     message: string;
     title?: string;
   } | null>(null);
 
   const showToast = useCallback(
-    (type: TToastType, message: string, title?: string) => {
+    (type: TToastTypes, message: string, title?: string) => {
       setToast({ type, message: translate(message), title });
     },
     [],

src/design/components/overlay/Toast.tsx

@@ -6,14 +6,14 @@
 import React, { useEffect } from "react";
 import { iconsLib } from "../../../assets";
 import { cn } from "../../helpers";
-import { ToastTypes, type TToastType } from "../../constants";
+import { ToastTypes, type TToastTypes } from "../../constants";
 
 export { ToastTypes };
-export type { TToastType };
+export type { TToastTypes };
 
 export interface IToastProps {
   message: string;
-  type?: TToastType;
+  type?: TToastTypes;
   duration?: number;
   onClose: () => void;
   icon?: React.ReactNode;
@@ -35,14 +35,14 @@ export const Toast: React.FC<IToastProps> = ({
     }
   }, [duration, onClose]);
 
-  const typeClasses = {
+  const typeClasses: Record<TToastTypes, string> = {
     success: "alert-success",
     info: "alert-info",
     warning: "alert-warning",
     error: "alert-error",
   };
 
-  const defaultIcons = {
+  const defaultIcons: Record<TToastTypes, React.ReactNode> = {
     success: (
       <iconsLib.check className="h-6 w-6 text-emerald-600 dark:text-emerald-400 shrink-0" />
     ),
@@ -57,7 +57,7 @@ export const Toast: React.FC<IToastProps> = ({
     ),
   };
 
-  const typeColors = {
+  const typeColors: Record<TToastTypes, string> = {
     success: "text-emerald-800 dark:text-emerald-200",
     info: "text-sky-800 dark:text-sky-200",
     warning: "text-amber-800 dark:text-amber-200",

src/design/constants/toast.constants.ts

@@ -11,4 +11,4 @@ export const ToastTypes = {
   ERROR: "error",
 } as const;
 
-export type TToastType = (typeof ToastTypes)[keyof typeof ToastTypes];
+export type TToastTypes = (typeof ToastTypes)[keyof typeof ToastTypes];

src/helpers/index.ts

@@ -1,3 +1,4 @@
 export * from "./jwt.helper";
 export * from "./date.helper";
+export * from "./socket.helpers.ts";
 export * from "../design/helpers";

src/helpers/socket.helpers.ts

@@ -0,0 +1,120 @@
+import { ToastTypes, type TToastTypes } from "../design";
+import { NOTIFICATION_SOCKET_TYPES } from "../modules/notification";
+
+export type ISocketMessage = {
+  type: string;
+  message?: string;
+  data?: Record<string, unknown>;
+  created_at?: string;
+  channel?: string;
+};
+
+export const SOCKET_MESSAGE_TYPES = {
+  NOTIFICATION: "notification",
+  SPEECH: "speech",
+} as const;
+
+export const SOCKET_CHANNELS = {
+  NOTIFICATION: "NotificationChannel",
+  SPEECH_LIVE: "SpeechLiveChannel",
+} as const;
+
+export const SPEECH_EVENT_TYPES = {
+  PARTIAL: "partial",
+  FINAL: "final",
+  ERROR: "error",
+} as const;
+
+export const SOCKET_SUBSCRIBE_TIMEOUT_MS = 10_000;
+export const SOCKET_CONNECT_WAIT_MS = 5_000;
+
+export const NOTIFICATION_TOAST_SUCCESS_TYPES = [
+  NOTIFICATION_SOCKET_TYPES.PAYMENT_SUCCESS,
+  NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CREATED,
+  NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_RESUMED,
+  NOTIFICATION_SOCKET_TYPES.WELCOME,
+  NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_READY,
+  NOTIFICATION_SOCKET_TYPES.TTS_READY,
+  NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSED,
+] as const;
+
+export const NOTIFICATION_TOAST_ERROR_TYPES = [
+  NOTIFICATION_SOCKET_TYPES.PAYMENT_FAILED,
+  NOTIFICATION_SOCKET_TYPES.SUBSCRIPTION_CANCELED,
+  NOTIFICATION_SOCKET_TYPES.AI_RESPONSE_FAILED,
+  NOTIFICATION_SOCKET_TYPES.TTS_FAILED,
+  NOTIFICATION_SOCKET_TYPES.ASSET_COMPRESSION_FAILED,
+] as const;
+
+export function parseCableChannel(identifier: unknown): string {
+  if (identifier && typeof identifier === "object") {
+    const channel = (identifier as { channel?: unknown }).channel;
+    return typeof channel === "string" ? channel : "";
+  }
+
+  if (typeof identifier !== "string" || identifier.trim() === "") {
+    return "";
+  }
+
+  try {
+    const parsed = JSON.parse(identifier) as { channel?: unknown };
+    return typeof parsed.channel === "string" ? parsed.channel : "";
+  } catch {
+    return "";
+  }
+}
+
+export function getSpeechEventType(data: ISocketMessage): string {
+  return typeof data.data?.type === "string" ? data.data.type : "";
+}
+
+export function isSpeechLiveMessage(data: ISocketMessage): boolean {
+  if (data.channel === SOCKET_CHANNELS.SPEECH_LIVE) {
+    return true;
+  }
+
+  if (data.type === SOCKET_MESSAGE_TYPES.SPEECH) {
+    const eventType = getSpeechEventType(data);
+    return (
+      eventType === SPEECH_EVENT_TYPES.PARTIAL ||
+      eventType === SPEECH_EVENT_TYPES.FINAL ||
+      eventType === SPEECH_EVENT_TYPES.ERROR
+    );
+  }
+
+  return false;
+}
+
+export function getSocketToast(data: ISocketMessage): {
+  kind: TToastTypes;
+  message: string;
+} | null {
+  const message = data.message?.trim() ?? "";
+
+  if (isSpeechLiveMessage(data)) {
+    if (getSpeechEventType(data) === SPEECH_EVENT_TYPES.ERROR && message) {
+      return { kind: ToastTypes.ERROR, message };
+    }
+    return null;
+  }
+
+  if (data.type !== SOCKET_MESSAGE_TYPES.NOTIFICATION || !message) {
+    return null;
+  }
+
+  const eventType = getSpeechEventType(data);
+
+  if (
+    (NOTIFICATION_TOAST_SUCCESS_TYPES as readonly string[]).includes(eventType)
+  ) {
+    return { kind: ToastTypes.SUCCESS, message };
+  }
+
+  if (
+    (NOTIFICATION_TOAST_ERROR_TYPES as readonly string[]).includes(eventType)
+  ) {
+    return { kind: ToastTypes.ERROR, message };
+  }
+
+  return { kind: ToastTypes.INFO, message };
+}