From e934b027412e286550581d9e0acb27fc5590c734 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Thu, 17 Sep 2026 14:23:22 +0000 Subject: [PATCH 1/4] feat: add browser-native text-to-speech for assistant replies Allows hearing assistant messages aloud using browser-native SpeechSynthesis: - Speaker action button on completed assistant replies matching sibling action buttons - Client-side SpeechSynthesis hook with garbage-collection retention and cross-component state sync - Markdown/code-fence sanitization so code blocks and syntax are not read aloud - Auto-read toggle and speech voice selector in Settings -> General - Full i18n localization for English, Japanese, and Simplified Chinese --- components/ChatWindow.tsx | 31 +++- components/MessageView.tsx | 28 +++- components/SettingsConfig.tsx | 37 +++++ hooks/useSpeechSynthesis.ts | 270 ++++++++++++++++++++++++++++++++++ lib/i18n/locales/en.json | 9 +- lib/i18n/locales/ja.json | 9 +- lib/i18n/locales/zh-CN.json | 9 +- lib/speech-sanitizer.test.mjs | 51 +++++++ lib/speech-sanitizer.ts | 51 +++++++ 9 files changed, 489 insertions(+), 6 deletions(-) create mode 100644 hooks/useSpeechSynthesis.ts create mode 100644 lib/speech-sanitizer.test.mjs create mode 100644 lib/speech-sanitizer.ts diff --git a/components/ChatWindow.tsx b/components/ChatWindow.tsx index 1a847ef1..f1bfe1d4 100644 --- a/components/ChatWindow.tsx +++ b/components/ChatWindow.tsx @@ -17,6 +17,7 @@ import OmpWebLogo from "./OmpWebLogo"; import { CHAT_COLUMN_MAX_WIDTH, MINIMAP_WIDTH } from "@/lib/chat-layout"; import { useAgentSession, type AgentPhase, type NoticeItem, type SubagentInfo } from "@/hooks/useAgentSession"; import { useAudio } from "@/hooks/useAudio"; +import { useSpeechSynthesis } from "@/hooks/useSpeechSynthesis"; import { useDragDrop } from "@/hooks/useDragDrop"; import { useIsMobile } from "@/hooks/useIsMobile"; import type { SessionStatsInfo, GenerationSpeedInfo } from "@/lib/pi-types"; @@ -530,11 +531,37 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa // checks the sound preference itself. const playDoneSoundRef = useRef(playDoneSound); playDoneSoundRef.current = playDoneSound; + const tts = useSpeechSynthesis(); + const ttsRef = useRef(tts); + useEffect(() => { ttsRef.current = tts; }, [tts]); + const messagesRef = useRef([]); + const entryIdsRef = useRef([]); + const wrappedOnAgentEnd = useCallback(() => { playDoneSoundRef.current(); + if (ttsRef.current.autoPlayEnabled) { + const msgs = messagesRef.current; + for (let i = msgs.length - 1; i >= 0; i--) { + const msg = msgs[i]; + if (msg.role === "assistant" && Array.isArray(msg.content)) { + const text = msg.content + .filter((b: unknown): b is { type: "text"; text: string } => { + if (!b || typeof b !== "object") return false; + if (!("type" in b) || b.type !== "text") return false; + return "text" in b && typeof b.text === "string"; + }) + .map((b) => b.text) + .join("\n\n"); + if (text.trim()) { + const speechId = entryIdsRef.current[i] ?? (msg.timestamp ? String(msg.timestamp) : "msg"); + ttsRef.current.speak(speechId, text); + } + break; + } + } + } onAgentEnd?.(); }, [onAgentEnd]); - // Stabilize the onEditContent ref; pairs with React.memo to avoid re-rendering history messages const handleEditContent = useCallback((content: string) => { chatInputRef?.current?.insertIfEmpty(content); @@ -566,6 +593,8 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa modelsRefreshKey, chatInputRef, onBranchDataChange, onSystemPromptChange, onSystemPromptLoaderChange, onSessionStatsPanelOpen, onOpenFile, }); + useEffect(() => { messagesRef.current = messages; }, [messages]); + useEffect(() => { entryIdsRef.current = entryIds; }, [entryIds]); const sessionBusy = agentRunning || bashRunning; const modelCapacity = useMemo(() => { if (!displayModelValue) return null; diff --git a/components/MessageView.tsx b/components/MessageView.tsx index c08ee528..e40eee46 100644 --- a/components/MessageView.tsx +++ b/components/MessageView.tsx @@ -1,9 +1,10 @@ "use client"; import { memo, useState, useId, useRef, useEffect, useMemo, useCallback, type ComponentProps } from "react"; -import { Copy, Check, GitFork, CornerUpLeft, ChevronRight, ChevronDown, Brain, EyeOff, CircleAlert, CircleSlash, LoaderCircle, FileText, Search, FileEdit, Terminal, CheckSquare, Bot, Code2, Globe, MessagesSquare, Wrench } from "lucide-react"; +import { Copy, Check, GitFork, CornerUpLeft, ChevronRight, ChevronDown, Brain, EyeOff, CircleAlert, CircleSlash, LoaderCircle, FileText, Search, FileEdit, Terminal, CheckSquare, Bot, Code2, Globe, MessagesSquare, Wrench, Volume2, Square } from "lucide-react"; import { MarkdownBody } from "./MarkdownBody"; import { MessageCopyActions } from "./MessageCopyActions"; +import { useSpeechSynthesis } from "@/hooks/useSpeechSynthesis"; import { ClickableImage } from "./ImageLightbox"; import { translate, useI18n, type Locale } from "@/lib/i18n"; import { parseCompactionSummary } from "@/lib/compaction-summary"; @@ -510,6 +511,15 @@ function AssistantMessageView({ liveTokensPerSecond?: number | null; }) { const { t, locale } = useI18n(); + const { isSupported: ttsSupported, isSpeaking: ttsSpeaking, speakingId: ttsSpeakingId, toggle: ttsToggle } = useSpeechSynthesis(); + const speakableText = useMemo(() => { + return (message.content ?? []) + .filter((b): b is TextContent => b.type === "text" && typeof b.text === "string") + .map((b) => b.text) + .join("\n\n"); + }, [message.content]); + const messageSpeechId = entryId ?? (message.timestamp ? String(message.timestamp) : "msg"); + const isThisSpeaking = ttsSpeaking && ttsSpeakingId === messageSpeechId; const time = showTimestamp ? formatTime(message.timestamp, locale) : null; const bodyRef = useRef(null); const texts = (message.content ?? []).filter((block): block is TextContent => block.type === "text").map((block) => block.text); @@ -735,10 +745,24 @@ function AssistantMessageView({ )} - {!isStreaming && (texts.some((text) => text.trim()) || time || canFork) && ( + {!isStreaming && (texts.some((text) => text.trim()) || time || canFork || (ttsSupported && speakableText.trim().length > 0)) && (
+ {ttsSupported && speakableText.trim().length > 0 && ( + + + + )} {canFork && }
{time && {time}} diff --git a/components/SettingsConfig.tsx b/components/SettingsConfig.tsx index 4506e6af..39410347 100644 --- a/components/SettingsConfig.tsx +++ b/components/SettingsConfig.tsx @@ -13,6 +13,7 @@ import { copyText } from "@/lib/clipboard"; import type { AppUpdateInfo } from "./AppUpdateDialog"; import { useFontSize, type FontSizePreference } from "@/hooks/useFontSize"; import { useUiScale, type UiScalePreference } from "@/hooks/useUiScale"; +import { useSpeechSynthesis } from "@/hooks/useSpeechSynthesis"; const SettingsTabLoading = () => { const { t } = useI18n(); return
{t("settingsConfig.loadingSettings")}
; @@ -131,6 +132,8 @@ const SETTING_INDEX: SettingIndexEntry[] = [ { id: "completion-sound", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.completionSound", descKey: "settingsConfig.completionSoundDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Completion sound", fallbackDesc: "Play a tone when the agent completes a run.", scope: "UI" }, { id: "keep-tool-calls-collapsed", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.keepToolCallsCollapsed", descKey: "settingsConfig.keepToolCallsCollapsedDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Keep tool calls collapsed", fallbackDesc: "Show only compact headers while tools execute.", scope: "UI" }, { id: "scope-native-select-all", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.scopeNativeSelectAll", descKey: "settingsConfig.scopeNativeSelectAllDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Scope native Select All (experimental)", fallbackDesc: "Limit whole-page selections from browser or touch menus to the active message, chat, or file. May also narrow deliberate whole-page selections. Turn off if selection handles or menus misbehave. Keyboard shortcuts are unaffected.", scope: "UI" }, + { id: "tts-autoplay", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.ttsAutoplay", descKey: "settingsConfig.ttsAutoplayDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Auto-read assistant responses", fallbackDesc: "Automatically read aloud new assistant replies when completed.", scope: "UI" }, + { id: "tts-voice", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.ttsVoice", descKey: "settingsConfig.ttsVoiceDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Speech Voice", fallbackDesc: "Select the browser voice for text-to-speech reading.", scope: "UI" }, { id: "provider-usage", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.providerUsage", descKey: "settingsConfig.providerUsageDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Provider usage limits", fallbackDesc: "Show provider usage in the sidebar, above Settings.", scope: "UI" }, { id: "chat-font-size", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.chatFontSize", descKey: "settingsConfig.chatFontSizeDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Chat Font Size", fallbackDesc: "Adjust text size for conversation messages, code blocks, and markdown output.", scope: "UI" }, { id: "ui-scale", tab: "general", sectionKey: "settingsConfig.interfaceBehavior", labelKey: "settingsConfig.uiScale", descKey: "settingsConfig.uiScaleDesc", fallbackSection: "Interface & Behavior", fallbackLabel: "Interface Scale", fallbackDesc: "Adjust overall UI zoom and display density across sidebars, dialogs, buttons, and toolbars.", scope: "UI" }, @@ -380,6 +383,14 @@ export function SettingsConfig({ activeTab, toolCallsDefaultCollapsed, onToolCal const workspaceReady = cwd !== null; const { fontSize, setFontSize } = useFontSize(); const { uiScale, setUiScale } = useUiScale(); + const { + isSupported: ttsSupported, + autoPlayEnabled: ttsAutoPlay, + setAutoPlay: setTtsAutoPlay, + voices: ttsVoices, + selectedVoiceURI: ttsVoiceURI, + setSelectedVoiceURI: setTtsVoiceURI, + } = useSpeechSynthesis(); const [searchQuery, setSearchQuery] = useState(""); const [highlightId, setHighlightId] = useState(null); const [submitBehavior, setSubmitBehavior] = useState(() => getSubmitDuringRunBehavior()); @@ -776,6 +787,32 @@ export function SettingsConfig({ activeTab, toolCallsDefaultCollapsed, onToolCal }} /> + {ttsSupported && ( + <> + + + + {ttsVoices.length > 0 && ( + + + + )} + + )} diff --git a/hooks/useSpeechSynthesis.ts b/hooks/useSpeechSynthesis.ts new file mode 100644 index 00000000..30de4829 --- /dev/null +++ b/hooks/useSpeechSynthesis.ts @@ -0,0 +1,270 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { sanitizeTextForSpeech } from "@/lib/speech-sanitizer"; + +export const TTS_AUTOPLAY_PREF_KEY = "omp-tts-autoplay"; +export const TTS_VOICE_PREF_KEY = "omp-tts-voice"; +export const TTS_PREF_EVENT = "omp-tts-pref-change"; +export const TTS_STATE_EVENT = "omp-tts-state-change"; + +export interface SpeechSynthesisState { + isSupported: boolean; + isSpeaking: boolean; + speakingId: string | null; + voices: SpeechSynthesisVoice[]; + selectedVoiceURI: string | null; + autoPlayEnabled: boolean; + speak: (id: string, text: string) => void; + stop: () => void; + toggle: (id: string, text: string) => void; + setAutoPlay: (enabled: boolean) => void; + setSelectedVoiceURI: (uri: string | null) => void; +} + +// Module-level reference to the active utterance to prevent Chromium GC bug +let activeGlobalUtterance: SpeechSynthesisUtterance | null = null; +let currentSpeakingId: string | null = null; + +export function getActiveUtterance(): SpeechSynthesisUtterance | null { + return activeGlobalUtterance; +} + +function broadcastState(speakingId: string | null, isSpeaking: boolean) { + currentSpeakingId = speakingId; + if (typeof window !== "undefined") { + window.dispatchEvent( + new CustomEvent(TTS_STATE_EVENT, { detail: { speakingId, isSpeaking } }) + ); + } +} + +export function useSpeechSynthesis(): SpeechSynthesisState { + const [isSupported, setIsSupported] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(() => currentSpeakingId !== null); + const [speakingId, setSpeakingId] = useState(() => currentSpeakingId); + const [voices, setVoices] = useState([]); + const [selectedVoiceURI, setSelectedVoiceURIState] = useState(() => { + if (typeof window === "undefined") return null; + try { + return localStorage.getItem(TTS_VOICE_PREF_KEY) || null; + } catch { + return null; + } + }); + const [autoPlayEnabled, setAutoPlayEnabledState] = useState(() => { + if (typeof window === "undefined") return false; + try { + return localStorage.getItem(TTS_AUTOPLAY_PREF_KEY) === "true"; + } catch { + return false; + } + }); + + const voicesRef = useRef([]); + const selectedVoiceURIRef = useRef(selectedVoiceURI); + + useEffect(() => { + voicesRef.current = voices; + }, [voices]); + + useEffect(() => { + selectedVoiceURIRef.current = selectedVoiceURI; + }, [selectedVoiceURI]); + + // Initial browser support check and voice loading + useEffect(() => { + if (typeof window === "undefined" || !("speechSynthesis" in window)) { + setIsSupported(false); + return; + } + + setIsSupported(true); + + const updateVoices = () => { + try { + const available = window.speechSynthesis.getVoices() || []; + setVoices(available); + } catch { + setVoices([]); + } + }; + + updateVoices(); + + if (window.speechSynthesis.onvoiceschanged !== undefined) { + window.speechSynthesis.onvoiceschanged = updateVoices; + } + + return () => { + if (typeof window !== "undefined" && "speechSynthesis" in window) { + window.speechSynthesis.onvoiceschanged = null; + } + }; + }, []); + + // Listen to global TTS state changes across components + useEffect(() => { + if (typeof window === "undefined") return; + + const handleStateChange = (e: Event) => { + const detail = (e as CustomEvent<{ speakingId: string | null; isSpeaking: boolean }>).detail; + if (!detail) return; + setSpeakingId(detail.speakingId); + setIsSpeaking(detail.isSpeaking); + }; + + window.addEventListener(TTS_STATE_EVENT, handleStateChange); + return () => window.removeEventListener(TTS_STATE_EVENT, handleStateChange); + }, []); + + // Sync preference changes across components + useEffect(() => { + if (typeof window === "undefined") return; + + const handlePrefChange = (e: Event) => { + const detail = (e as CustomEvent<{ autoPlay?: boolean; voiceURI?: string | null }>).detail; + if (!detail) return; + if (typeof detail.autoPlay === "boolean") { + setAutoPlayEnabledState(detail.autoPlay); + } + if (detail.voiceURI !== undefined) { + setSelectedVoiceURIState(detail.voiceURI); + } + }; + + window.addEventListener(TTS_PREF_EVENT, handlePrefChange); + return () => window.removeEventListener(TTS_PREF_EVENT, handlePrefChange); + }, []); + + const stop = useCallback(() => { + if (typeof window === "undefined" || !("speechSynthesis" in window)) return; + try { + window.speechSynthesis.cancel(); + } catch { + // ignore + } + activeGlobalUtterance = null; + broadcastState(null, false); + }, []); + + // Stop playback on Escape key + useEffect(() => { + if (typeof window === "undefined") return; + + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape" && currentSpeakingId !== null) { + stop(); + } + }; + + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [stop]); + + const speak = useCallback( + (id: string, text: string) => { + if (typeof window === "undefined" || !("speechSynthesis" in window)) return; + + const cleanText = sanitizeTextForSpeech(text); + if (!cleanText) return; + + try { + window.speechSynthesis.cancel(); + } catch { + // ignore + } + + const utterance = new SpeechSynthesisUtterance(cleanText); + activeGlobalUtterance = utterance; + + const voice = voicesRef.current.find((v) => v.voiceURI === selectedVoiceURIRef.current); + if (voice) { + utterance.voice = voice; + } + + utterance.onstart = () => { + broadcastState(id, true); + }; + + utterance.onend = () => { + activeGlobalUtterance = null; + broadcastState(null, false); + }; + + utterance.onerror = (e) => { + if (e.error !== "canceled" && e.error !== "interrupted") { + console.warn("SpeechSynthesis error:", e.error); + } + activeGlobalUtterance = null; + broadcastState(null, false); + }; + + try { + window.speechSynthesis.speak(utterance); + } catch (err) { + console.warn("Failed to speak utterance:", err); + activeGlobalUtterance = null; + broadcastState(null, false); + } + }, + [] + ); + + const toggle = useCallback( + (id: string, text: string) => { + if (currentSpeakingId === id) { + stop(); + } else { + speak(id, text); + } + }, + [speak, stop] + ); + + const setAutoPlay = useCallback((enabled: boolean) => { + setAutoPlayEnabledState(enabled); + if (typeof window !== "undefined") { + try { + localStorage.setItem(TTS_AUTOPLAY_PREF_KEY, String(enabled)); + } catch { + // ignore + } + window.dispatchEvent( + new CustomEvent(TTS_PREF_EVENT, { detail: { autoPlay: enabled } }) + ); + } + }, []); + + const setSelectedVoiceURI = useCallback((uri: string | null) => { + setSelectedVoiceURIState(uri); + if (typeof window !== "undefined") { + try { + if (uri) { + localStorage.setItem(TTS_VOICE_PREF_KEY, uri); + } else { + localStorage.removeItem(TTS_VOICE_PREF_KEY); + } + } catch { + // ignore + } + window.dispatchEvent( + new CustomEvent(TTS_PREF_EVENT, { detail: { voiceURI: uri } }) + ); + } + }, []); + + return { + isSupported, + isSpeaking, + speakingId, + voices, + selectedVoiceURI, + autoPlayEnabled, + speak, + stop, + toggle, + setAutoPlay, + setSelectedVoiceURI, + }; +} diff --git a/lib/i18n/locales/en.json b/lib/i18n/locales/en.json index ae2e0f91..aa218d0c 100644 --- a/lib/i18n/locales/en.json +++ b/lib/i18n/locales/en.json @@ -1412,5 +1412,12 @@ "usageConfig.emptyTitle": "No usage records found", "usageConfig.emptyDesc": "Start chatting with models to see your token analytics and costs here.", "usageConfig.loading": "Loading usage analytics…", - "usageConfig.error": "Could not load usage data" + "usageConfig.error": "Could not load usage data", + "settingsConfig.ttsAutoplay": "Auto-read assistant responses", + "settingsConfig.ttsAutoplayDesc": "Automatically read aloud new assistant replies when completed.", + "settingsConfig.ttsVoice": "Speech Voice", + "settingsConfig.ttsVoiceDesc": "Select the browser voice for text-to-speech reading.", + "settingsConfig.defaultVoice": "Default system voice", + "messageView.readAloud": "Read aloud", + "messageView.stopSpeech": "Stop speaking" } diff --git a/lib/i18n/locales/ja.json b/lib/i18n/locales/ja.json index 29b7362e..e25f41f1 100644 --- a/lib/i18n/locales/ja.json +++ b/lib/i18n/locales/ja.json @@ -1383,5 +1383,12 @@ "usageConfig.emptyTitle": "利用レコードが見つかりません", "usageConfig.emptyDesc": "モデルと対話を開始すると、ここにトークン分析とコストが表示されます。", "usageConfig.loading": "利用状況を読み込み中…", - "usageConfig.error": "利用状況データを読み込めませんでした" + "usageConfig.error": "利用状況データを読み込めませんでした", + "settingsConfig.ttsAutoplay": "アシスタント応答の自動読み上げ", + "settingsConfig.ttsAutoplayDesc": "完了時にアシスタントの新しい応答を自動的に音声で読み上げます。", + "settingsConfig.ttsVoice": "読み上げ音声", + "settingsConfig.ttsVoiceDesc": "テキスト読み上げに使用するブラウザの音声を選択します。", + "settingsConfig.defaultVoice": "システムの既定の音声", + "messageView.readAloud": "読み上げ", + "messageView.stopSpeech": "読み上げを停止" } diff --git a/lib/i18n/locales/zh-CN.json b/lib/i18n/locales/zh-CN.json index 7b46529d..93b43f9f 100644 --- a/lib/i18n/locales/zh-CN.json +++ b/lib/i18n/locales/zh-CN.json @@ -1383,5 +1383,12 @@ "usageConfig.emptyTitle": "暂无用量记录", "usageConfig.emptyDesc": "开始与模型对话后，将在此处显示 Token 用量和费用分析。", "usageConfig.loading": "正在加载用量分析…", - "usageConfig.error": "加载用量数据失败" + "usageConfig.error": "加载用量数据失败", + "settingsConfig.ttsAutoplay": "自动朗读助手回复", + "settingsConfig.ttsAutoplayDesc": "生成完成后自动语音朗读助手的新回复。", + "settingsConfig.ttsVoice": "朗读语音", + "settingsConfig.ttsVoiceDesc": "选择用于文本转语音朗读的浏览器声音。", + "settingsConfig.defaultVoice": "系统默认声音", + "messageView.readAloud": "朗读", + "messageView.stopSpeech": "停止朗读" } diff --git a/lib/speech-sanitizer.test.mjs b/lib/speech-sanitizer.test.mjs new file mode 100644 index 00000000..5d2cc023 --- /dev/null +++ b/lib/speech-sanitizer.test.mjs @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createJiti } from "jiti"; + +const jiti = createJiti(import.meta.url); +const { sanitizeTextForSpeech } = await jiti.import("./speech-sanitizer.ts"); + +test("sanitizeTextForSpeech handles empty or falsy inputs", () => { + assert.equal(sanitizeTextForSpeech(""), ""); + assert.equal(sanitizeTextForSpeech(null), ""); + assert.equal(sanitizeTextForSpeech(undefined), ""); +}); + +test("sanitizeTextForSpeech strips fenced code blocks", () => { + const input = "Here is the code:\n```typescript\nconst x = 10;\nconsole.log(x);\n```\nLet me know what you think."; + const expected = "Here is the code: Let me know what you think."; + assert.equal(sanitizeTextForSpeech(input), expected); +}); + +test("sanitizeTextForSpeech preserves inline code text without backticks", () => { + const input = "Run `npm run dev` to start."; + const expected = "Run npm run dev to start."; + assert.equal(sanitizeTextForSpeech(input), expected); +}); + +test("sanitizeTextForSpeech unwraps markdown links", () => { + const input = "Check out the [documentation](https://example.com/docs) for details."; + const expected = "Check out the documentation for details."; + assert.equal(sanitizeTextForSpeech(input), expected); +}); + +test("sanitizeTextForSpeech strips images and bare URLs", () => { + const input = "Look at this ![diagram](https://example.com/img.png) and visit https://example.com directly."; + const expected = "Look at this and visit directly."; + assert.equal(sanitizeTextForSpeech(input), expected); +}); + +test("sanitizeTextForSpeech strips markdown formatting headers, bold, italics, lists, and blockquotes", () => { + const input = ` +# Title +> Important note: + +- First **bold** point +- Second *italic* point +- Third ~~strike~~ point + +1. Numbered step +`; + const result = sanitizeTextForSpeech(input); + assert.equal(result, "Title Important note: First bold point Second italic point Third strike point Numbered step"); +}); diff --git a/lib/speech-sanitizer.ts b/lib/speech-sanitizer.ts new file mode 100644 index 00000000..ceb4eb1f --- /dev/null +++ b/lib/speech-sanitizer.ts @@ -0,0 +1,51 @@ +/** + * Sanitizes markdown / formatted text for clean speech synthesis. + * Strips code blocks, markdown syntax, links, and URLs so spoken output sounds natural. + */ +export function sanitizeTextForSpeech(text: string): string { + if (!text) return ""; + + let cleaned = text; + + // 1. Remove fenced code blocks completely (reading raw syntax or long code in TTS is unlistenable) + cleaned = cleaned.replace(/```[\s\S]*?```/g, " "); + + // 2. Remove inline code backticks, keeping inner content + cleaned = cleaned.replace(/`([^`]+)`/g, "$1"); + + // 3. Remove images ![alt](url) + cleaned = cleaned.replace(/!\[([^\]]*)\]\([^)]*\)/g, ""); + + // 4. Convert markdown links [text](url) -> text + cleaned = cleaned.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1"); + + // 5. Remove bare URLs + cleaned = cleaned.replace(/https?:\/\/\S+/g, ""); + + // 6. Remove HTML tags + cleaned = cleaned.replace(/<[^>]+>/g, ""); + + // 7. Remove headers (# Header -> Header) + cleaned = cleaned.replace(/^#{1,6}\s+/gm, ""); + + // 8. Remove blockquotes (> quote -> quote) + cleaned = cleaned.replace(/^>\s+/gm, ""); + + // 9. Remove bold / italic markers (***text***, **text**, *text*, __text__, _text_) + cleaned = cleaned.replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1"); + + // 10. Remove strikethrough (~~text~~ -> text) + cleaned = cleaned.replace(/~~([^~]+)~~/g, "$1"); + + // 11. Remove markdown bullet points / list numbers (- item, * item, 1. item) + cleaned = cleaned.replace(/^[\s]*[-*+]\s+/gm, ""); + cleaned = cleaned.replace(/^[\s]*\d+\.\s+/gm, ""); + + // 12. Remove horizontal rules (---, ***, ___) + cleaned = cleaned.replace(/^[-*_]{3,}\s*$/gm, ""); + + // 13. Collapse multiple whitespace and newlines + cleaned = cleaned.replace(/\s+/g, " ").trim(); + + return cleaned; +} From b68ab7183b4e7ed8fbeb5967ffb97d79d4c14f0e Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Fri, 18 Sep 2026 05:36:38 +0000 Subject: [PATCH 2/4] fix(tts): address copilot review findings - Use addEventListener for voiceschanged so per-hook mounts stop clobbering the shared handler - Mount the speech hook once via SpeechSynthesisProvider; transcript rows consume useSpeechContext - Autoplay reads streamState.streamingMessage with assistant-role narrowing instead of stale messagesRef - TTS settings render disabled with an explanation on unsupported browsers instead of hiding - Add settingsConfig.ttsNotSupported localization (en/ja/zh-CN) --- components/ChatWindow.tsx | 60 +++++++++++++------ components/MessageView.tsx | 4 +- components/SettingsConfig.tsx | 58 ++++++++++-------- ...echSynthesis.ts => useSpeechSynthesis.tsx} | 25 ++++++-- lib/i18n/locales/en.json | 3 +- lib/i18n/locales/ja.json | 3 +- lib/i18n/locales/zh-CN.json | 3 +- 7 files changed, 101 insertions(+), 55 deletions(-) rename hooks/{useSpeechSynthesis.ts => useSpeechSynthesis.tsx} (90%) diff --git a/components/ChatWindow.tsx b/components/ChatWindow.tsx index f1bfe1d4..1d66473b 100644 --- a/components/ChatWindow.tsx +++ b/components/ChatWindow.tsx @@ -17,7 +17,7 @@ import OmpWebLogo from "./OmpWebLogo"; import { CHAT_COLUMN_MAX_WIDTH, MINIMAP_WIDTH } from "@/lib/chat-layout"; import { useAgentSession, type AgentPhase, type NoticeItem, type SubagentInfo } from "@/hooks/useAgentSession"; import { useAudio } from "@/hooks/useAudio"; -import { useSpeechSynthesis } from "@/hooks/useSpeechSynthesis"; +import { useSpeechSynthesis, SpeechSynthesisProvider } from "@/hooks/useSpeechSynthesis"; import { useDragDrop } from "@/hooks/useDragDrop"; import { useIsMobile } from "@/hooks/useIsMobile"; import type { SessionStatsInfo, GenerationSpeedInfo } from "@/lib/pi-types"; @@ -536,29 +536,49 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa useEffect(() => { ttsRef.current = tts; }, [tts]); const messagesRef = useRef([]); const entryIdsRef = useRef([]); + const streamStateRef = useRef | null>(null); const wrappedOnAgentEnd = useCallback(() => { playDoneSoundRef.current(); if (ttsRef.current.autoPlayEnabled) { - const msgs = messagesRef.current; - for (let i = msgs.length - 1; i >= 0; i--) { - const msg = msgs[i]; - if (msg.role === "assistant" && Array.isArray(msg.content)) { - const text = msg.content - .filter((b: unknown): b is { type: "text"; text: string } => { - if (!b || typeof b !== "object") return false; - if (!("type" in b) || b.type !== "text") return false; - return "text" in b && typeof b.text === "string"; - }) - .map((b) => b.text) - .join("\n\n"); - if (text.trim()) { - const speechId = entryIdsRef.current[i] ?? (msg.timestamp ? String(msg.timestamp) : "msg"); - ttsRef.current.speak(speechId, text); + const streamingMsg = streamStateRef.current; + let textToSpeak = ""; + let speechId = "msg"; + + if (streamingMsg && streamingMsg.role === "assistant" && Array.isArray(streamingMsg.content)) { + textToSpeak = streamingMsg.content + .filter((b: unknown): b is { type: "text"; text: string } => { + if (!b || typeof b !== "object") return false; + if (!("type" in b) || b.type !== "text") return false; + return "text" in b && typeof b.text === "string"; + }) + .map((b: { text: string }) => b.text) + .join("\n\n"); + speechId = streamingMsg.timestamp ? String(streamingMsg.timestamp) : "msg"; + } + + if (!textToSpeak.trim()) { + const msgs = messagesRef.current; + for (let i = msgs.length - 1; i >= 0; i--) { + const msg = msgs[i]; + if (msg.role === "assistant" && Array.isArray(msg.content)) { + textToSpeak = msg.content + .filter((b: unknown): b is { type: "text"; text: string } => { + if (!b || typeof b !== "object") return false; + if (!("type" in b) || b.type !== "text") return false; + return "text" in b && typeof b.text === "string"; + }) + .map((b) => b.text) + .join("\n\n"); + speechId = entryIdsRef.current[i] ?? (msg.timestamp ? String(msg.timestamp) : "msg"); + break; } - break; } } + + if (textToSpeak.trim()) { + ttsRef.current.speak(speechId, textToSpeak); + } } onAgentEnd?.(); }, [onAgentEnd]); @@ -595,6 +615,7 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa }); useEffect(() => { messagesRef.current = messages; }, [messages]); useEffect(() => { entryIdsRef.current = entryIds; }, [entryIds]); + useEffect(() => { streamStateRef.current = streamState.streamingMessage; }, [streamState]); const sessionBusy = agentRunning || bashRunning; const modelCapacity = useMemo(() => { if (!displayModelValue) return null; @@ -1107,8 +1128,8 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa
); } - return ( +
)} -
+ + ); } diff --git a/components/MessageView.tsx b/components/MessageView.tsx index e40eee46..e213421b 100644 --- a/components/MessageView.tsx +++ b/components/MessageView.tsx @@ -4,7 +4,7 @@ import { memo, useState, useId, useRef, useEffect, useMemo, useCallback, type Co import { Copy, Check, GitFork, CornerUpLeft, ChevronRight, ChevronDown, Brain, EyeOff, CircleAlert, CircleSlash, LoaderCircle, FileText, Search, FileEdit, Terminal, CheckSquare, Bot, Code2, Globe, MessagesSquare, Wrench, Volume2, Square } from "lucide-react"; import { MarkdownBody } from "./MarkdownBody"; import { MessageCopyActions } from "./MessageCopyActions"; -import { useSpeechSynthesis } from "@/hooks/useSpeechSynthesis"; +import { useSpeechContext } from "@/hooks/useSpeechSynthesis"; import { ClickableImage } from "./ImageLightbox"; import { translate, useI18n, type Locale } from "@/lib/i18n"; import { parseCompactionSummary } from "@/lib/compaction-summary"; @@ -511,7 +511,7 @@ function AssistantMessageView({ liveTokensPerSecond?: number | null; }) { const { t, locale } = useI18n(); - const { isSupported: ttsSupported, isSpeaking: ttsSpeaking, speakingId: ttsSpeakingId, toggle: ttsToggle } = useSpeechSynthesis(); + const { isSupported: ttsSupported, isSpeaking: ttsSpeaking, speakingId: ttsSpeakingId, toggle: ttsToggle } = useSpeechContext(); const speakableText = useMemo(() => { return (message.content ?? []) .filter((b): b is TextContent => b.type === "text" && typeof b.text === "string") diff --git a/components/SettingsConfig.tsx b/components/SettingsConfig.tsx index 39410347..ec6a3751 100644 --- a/components/SettingsConfig.tsx +++ b/components/SettingsConfig.tsx @@ -787,32 +787,38 @@ export function SettingsConfig({ activeTab, toolCallsDefaultCollapsed, onToolCal }} /> - {ttsSupported && ( - <> - - - - {ttsVoices.length > 0 && ( - - - - )} - - )} + + + + + + diff --git a/hooks/useSpeechSynthesis.ts b/hooks/useSpeechSynthesis.tsx similarity index 90% rename from hooks/useSpeechSynthesis.ts rename to hooks/useSpeechSynthesis.tsx index 30de4829..83fb3e95 100644 --- a/hooks/useSpeechSynthesis.ts +++ b/hooks/useSpeechSynthesis.tsx @@ -1,6 +1,6 @@ "use client"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { createContext, useCallback, useContext, useEffect, useRef, useState, type ReactNode } from "react"; import { sanitizeTextForSpeech } from "@/lib/speech-sanitizer"; export const TTS_AUTOPLAY_PREF_KEY = "omp-tts-autoplay"; @@ -92,13 +92,11 @@ export function useSpeechSynthesis(): SpeechSynthesisState { updateVoices(); - if (window.speechSynthesis.onvoiceschanged !== undefined) { - window.speechSynthesis.onvoiceschanged = updateVoices; - } + window.speechSynthesis.addEventListener("voiceschanged", updateVoices); return () => { if (typeof window !== "undefined" && "speechSynthesis" in window) { - window.speechSynthesis.onvoiceschanged = null; + window.speechSynthesis.removeEventListener("voiceschanged", updateVoices); } }; }, []); @@ -268,3 +266,20 @@ export function useSpeechSynthesis(): SpeechSynthesisState { setSelectedVoiceURI, }; } + +const SpeechSynthesisContext = createContext(null); + +export function SpeechSynthesisProvider({ children }: { children: ReactNode }) { + const speech = useSpeechSynthesis(); + return ( + + {children} + + ); +} + +export function useSpeechContext(): SpeechSynthesisState { + const ctx = useContext(SpeechSynthesisContext); + const fallback = useSpeechSynthesis(); + return ctx ?? fallback; +} diff --git a/lib/i18n/locales/en.json b/lib/i18n/locales/en.json index aa218d0c..bfc097eb 100644 --- a/lib/i18n/locales/en.json +++ b/lib/i18n/locales/en.json @@ -1419,5 +1419,6 @@ "settingsConfig.ttsVoiceDesc": "Select the browser voice for text-to-speech reading.", "settingsConfig.defaultVoice": "Default system voice", "messageView.readAloud": "Read aloud", - "messageView.stopSpeech": "Stop speaking" + "messageView.stopSpeech": "Stop speaking", + "settingsConfig.ttsNotSupported": "Not supported in this browser" } diff --git a/lib/i18n/locales/ja.json b/lib/i18n/locales/ja.json index e25f41f1..5da7d9df 100644 --- a/lib/i18n/locales/ja.json +++ b/lib/i18n/locales/ja.json @@ -1390,5 +1390,6 @@ "settingsConfig.ttsVoiceDesc": "テキスト読み上げに使用するブラウザの音声を選択します。", "settingsConfig.defaultVoice": "システムの既定の音声", "messageView.readAloud": "読み上げ", - "messageView.stopSpeech": "読み上げを停止" + "messageView.stopSpeech": "読み上げを停止", + "settingsConfig.ttsNotSupported": "このブラウザではサポートされていません" } diff --git a/lib/i18n/locales/zh-CN.json b/lib/i18n/locales/zh-CN.json index 93b43f9f..003ed9f9 100644 --- a/lib/i18n/locales/zh-CN.json +++ b/lib/i18n/locales/zh-CN.json @@ -1390,5 +1390,6 @@ "settingsConfig.ttsVoiceDesc": "选择用于文本转语音朗读的浏览器声音。", "settingsConfig.defaultVoice": "系统默认声音", "messageView.readAloud": "朗读", - "messageView.stopSpeech": "停止朗读" + "messageView.stopSpeech": "停止朗读", + "settingsConfig.ttsNotSupported": "当前浏览器不支持" } From 2b49718334f7fc6a9c80f4addebe7a297f3dda98 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Fri, 18 Sep 2026 10:05:25 +0000 Subject: [PATCH 3/4] fix(tts): address CodeRabbit review findings - speech sanitizer strips tag-shaped HTML only, so "x < y and z > 0" survives - utterance lifecycle callbacks identity-check the active utterance before mutating shared speech state - autoplay speaks from the render that commits the finished reply instead of refs that lag onAgentEnd - the provider owns the single speech controller; useSpeechContext no longer mounts one fallback controller per assistant row --- components/ChatWindow.tsx | 99 ++++++++++++++++++----------------- hooks/useSpeechSynthesis.tsx | 31 ++++++++--- lib/speech-sanitizer.test.mjs | 11 ++++ lib/speech-sanitizer.ts | 4 +- 4 files changed, 89 insertions(+), 56 deletions(-) diff --git a/components/ChatWindow.tsx b/components/ChatWindow.tsx index 1d66473b..d34b335f 100644 --- a/components/ChatWindow.tsx +++ b/components/ChatWindow.tsx @@ -93,6 +93,44 @@ function getUserInputText(message: AgentMessage): string | null { return text.length > 0 ? text : null; } +/** + * Text of the newest assistant reply for read-aloud. The live streaming + * message wins when present; otherwise the last committed assistant row is + * used, keyed by its entry id so the row's own speaker button lights up. + */ +function assistantSpeech( + messages: AgentMessage[], + entryIds: string[], + streaming: Partial | null, +): { id: string; text: string } | null { + const textOf = (content: unknown): string => { + if (!Array.isArray(content)) return ""; + return content + .filter((block: unknown): block is { type: "text"; text: string } => { + if (!block || typeof block !== "object") return false; + if (!("type" in block) || block.type !== "text") return false; + return "text" in block && typeof block.text === "string"; + }) + .map((block) => block.text) + .join("\n\n"); + }; + + if (streaming && streaming.role === "assistant") { + const text = textOf(streaming.content); + if (text.trim()) { + return { id: streaming.timestamp ? String(streaming.timestamp) : "msg", text }; + } + } + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (message.role !== "assistant") continue; + const text = textOf(message.content); + if (!text.trim()) break; + return { id: entryIds[i] ?? (message.timestamp ? String(message.timestamp) : "msg"), text }; + } + return null; +} + function withAssistantBlocks( message: AssistantMessage, content: AssistantContentBlock[], @@ -533,53 +571,15 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa playDoneSoundRef.current = playDoneSound; const tts = useSpeechSynthesis(); const ttsRef = useRef(tts); - useEffect(() => { ttsRef.current = tts; }, [tts]); - const messagesRef = useRef([]); - const entryIdsRef = useRef([]); - const streamStateRef = useRef | null>(null); + ttsRef.current = tts; + // omp calls onAgentEnd in the same tick as the state update that commits the + // finished reply, so reading the transcript here would still see the previous + // one. Flag it instead and speak from the render that carries it. + const autoplayPendingRef = useRef(false); const wrappedOnAgentEnd = useCallback(() => { playDoneSoundRef.current(); - if (ttsRef.current.autoPlayEnabled) { - const streamingMsg = streamStateRef.current; - let textToSpeak = ""; - let speechId = "msg"; - - if (streamingMsg && streamingMsg.role === "assistant" && Array.isArray(streamingMsg.content)) { - textToSpeak = streamingMsg.content - .filter((b: unknown): b is { type: "text"; text: string } => { - if (!b || typeof b !== "object") return false; - if (!("type" in b) || b.type !== "text") return false; - return "text" in b && typeof b.text === "string"; - }) - .map((b: { text: string }) => b.text) - .join("\n\n"); - speechId = streamingMsg.timestamp ? String(streamingMsg.timestamp) : "msg"; - } - - if (!textToSpeak.trim()) { - const msgs = messagesRef.current; - for (let i = msgs.length - 1; i >= 0; i--) { - const msg = msgs[i]; - if (msg.role === "assistant" && Array.isArray(msg.content)) { - textToSpeak = msg.content - .filter((b: unknown): b is { type: "text"; text: string } => { - if (!b || typeof b !== "object") return false; - if (!("type" in b) || b.type !== "text") return false; - return "text" in b && typeof b.text === "string"; - }) - .map((b) => b.text) - .join("\n\n"); - speechId = entryIdsRef.current[i] ?? (msg.timestamp ? String(msg.timestamp) : "msg"); - break; - } - } - } - - if (textToSpeak.trim()) { - ttsRef.current.speak(speechId, textToSpeak); - } - } + if (ttsRef.current.autoPlayEnabled) autoplayPendingRef.current = true; onAgentEnd?.(); }, [onAgentEnd]); // Stabilize the onEditContent ref; pairs with React.memo to avoid re-rendering history messages @@ -613,9 +613,12 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa modelsRefreshKey, chatInputRef, onBranchDataChange, onSystemPromptChange, onSystemPromptLoaderChange, onSessionStatsPanelOpen, onOpenFile, }); - useEffect(() => { messagesRef.current = messages; }, [messages]); - useEffect(() => { entryIdsRef.current = entryIds; }, [entryIds]); - useEffect(() => { streamStateRef.current = streamState.streamingMessage; }, [streamState]); + useEffect(() => { + if (!autoplayPendingRef.current) return; + autoplayPendingRef.current = false; + const speech = assistantSpeech(messages, entryIds, streamState.streamingMessage); + if (speech) ttsRef.current.speak(speech.id, speech.text); + }, [messages, entryIds, streamState, agentRunning]); const sessionBusy = agentRunning || bashRunning; const modelCapacity = useMemo(() => { if (!displayModelValue) return null; @@ -1129,7 +1132,7 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa ); } return ( - +
{ + if (activeGlobalUtterance !== utterance) return; broadcastState(id, true); }; utterance.onend = () => { + if (activeGlobalUtterance !== utterance) return; activeGlobalUtterance = null; broadcastState(null, false); }; @@ -194,6 +198,7 @@ export function useSpeechSynthesis(): SpeechSynthesisState { if (e.error !== "canceled" && e.error !== "interrupted") { console.warn("SpeechSynthesis error:", e.error); } + if (activeGlobalUtterance !== utterance) return; activeGlobalUtterance = null; broadcastState(null, false); }; @@ -269,17 +274,31 @@ export function useSpeechSynthesis(): SpeechSynthesisState { const SpeechSynthesisContext = createContext(null); -export function SpeechSynthesisProvider({ children }: { children: ReactNode }) { - const speech = useSpeechSynthesis(); +export function SpeechSynthesisProvider({ value, children }: { value: SpeechSynthesisState; children: ReactNode }) { return ( - + {children} ); } +// Outside a provider there is no controller to share. Returning an inert state +// keeps consumers renderable (the read-aloud action just stays hidden) without +// mounting one more controller — and its window listeners — per message row. +const INERT_SPEECH_STATE: SpeechSynthesisState = { + isSupported: false, + isSpeaking: false, + speakingId: null, + voices: [], + selectedVoiceURI: null, + autoPlayEnabled: false, + speak: () => {}, + stop: () => {}, + toggle: () => {}, + setAutoPlay: () => {}, + setSelectedVoiceURI: () => {}, +}; + export function useSpeechContext(): SpeechSynthesisState { - const ctx = useContext(SpeechSynthesisContext); - const fallback = useSpeechSynthesis(); - return ctx ?? fallback; + return useContext(SpeechSynthesisContext) ?? INERT_SPEECH_STATE; } diff --git a/lib/speech-sanitizer.test.mjs b/lib/speech-sanitizer.test.mjs index 5d2cc023..5ca043ff 100644 --- a/lib/speech-sanitizer.test.mjs +++ b/lib/speech-sanitizer.test.mjs @@ -35,6 +35,17 @@ test("sanitizeTextForSpeech strips images and bare URLs", () => { assert.equal(sanitizeTextForSpeech(input), expected); }); +test("sanitizeTextForSpeech strips html tags but keeps comparison operators", () => { + assert.equal( + sanitizeTextForSpeech("
Wrapped
text"), + "Wrapped text", + ); + assert.equal( + sanitizeTextForSpeech("Use x < y and z > 0 to guard."), + "Use x < y and z > 0 to guard.", + ); +}); + test("sanitizeTextForSpeech strips markdown formatting headers, bold, italics, lists, and blockquotes", () => { const input = ` # Title diff --git a/lib/speech-sanitizer.ts b/lib/speech-sanitizer.ts index ceb4eb1f..f40c1b03 100644 --- a/lib/speech-sanitizer.ts +++ b/lib/speech-sanitizer.ts @@ -22,8 +22,8 @@ export function sanitizeTextForSpeech(text: string): string { // 5. Remove bare URLs cleaned = cleaned.replace(/https?:\/\/\S+/g, ""); - // 6. Remove HTML tags - cleaned = cleaned.replace(/<[^>]+>/g, ""); + // 6. Remove HTML tags — tag-shaped only, so "x < y and z > 0" survives + cleaned = cleaned.replace(/<\/?[A-Za-z][^<>]*>/g, ""); // 7. Remove headers (# Header -> Header) cleaned = cleaned.replace(/^#{1,6}\s+/gm, ""); From 55d95c06b66007e1c637390dd5488ff002814410 Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Fri, 18 Sep 2026 12:14:13 +0000 Subject: [PATCH 4/4] fix(tts): synchronize ttsRef in useEffect instead of render --- components/ChatWindow.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/ChatWindow.tsx b/components/ChatWindow.tsx index d34b335f..621d7f31 100644 --- a/components/ChatWindow.tsx +++ b/components/ChatWindow.tsx @@ -571,7 +571,9 @@ export function ChatWindow({ session, newSessionCwd, newSessionWorkspace, toolCa playDoneSoundRef.current = playDoneSound; const tts = useSpeechSynthesis(); const ttsRef = useRef(tts); - ttsRef.current = tts; + useEffect(() => { + ttsRef.current = tts; + }, [tts]); // omp calls onAgentEnd in the same tick as the state update that commits the // finished reply, so reading the transcript here would still see the previous // one. Flag it instead and speak from the render that carries it.