From c941a13dcacd0d0f28d2aff1ce0fe74b9de1a481 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 26 Aug 2026 18:49:26 -0700 Subject: [PATCH 1/2] feat(chat): add live voice input waveform --- apps/desktop/src/main/ipc.test.ts | 22 +++- apps/desktop/src/main/ipc.ts | 36 +++++- apps/desktop/src/preload/index.ts | 2 + .../components/mic-button/mic-button.test.tsx | 112 ++++++++++++++++++ .../components/mic-button/mic-button.tsx | 109 +++++++++++++++-- .../home/components/user-input/user-input.tsx | 24 +++- apps/sim/hooks/use-speech-to-text.ts | 33 +++++- packages/desktop-bridge/contract-snapshot.ts | 28 +++++ packages/desktop-bridge/src/index.ts | 2 + 9 files changed, 351 insertions(+), 17 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.test.tsx diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index ba248a3f46b..11e942df59f 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -130,7 +130,7 @@ import { } from '@/main/browser-import' import { getSearchSuggestions } from '@/main/browser-search/suggestions' import { trackInputActivity } from '@/main/input-activity' -import { type IpcDeps, registerIpcHandlers } from '@/main/ipc' +import { type IpcDeps, openMicrophoneSettings, registerIpcHandlers } from '@/main/ipc' import { LocalFilesystemService } from '@/main/local-filesystem' import { TerminalRegistry } from '@/main/terminal/registry' import { findCachedTerminalThemeProfile, listTerminalThemeProfiles } from '@/main/terminal-themes' @@ -337,6 +337,26 @@ describe('registerIpcHandlers', () => { expect(shell.openExternal).toHaveBeenCalledTimes(1) }) + it('opens microphone privacy settings only for the trusted app origin', async () => { + const { invoke } = collectHandlers() + const handler = invoke.get('desktop:open-microphone-settings') + + expect(await handler?.(evilEvent)).toBe(false) + expect(await handler?.(appEvent)).toBe(process.platform === 'darwin') + expect(shell.openExternal).toHaveBeenCalledTimes(process.platform === 'darwin' ? 1 : 0) + }) + + it('uses fixed native microphone settings URLs', async () => { + await expect(openMicrophoneSettings('darwin')).resolves.toBe(true) + expect(shell.openExternal).toHaveBeenLastCalledWith( + 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone' + ) + + await expect(openMicrophoneSettings('win32')).resolves.toBe(true) + expect(shell.openExternal).toHaveBeenLastCalledWith('ms-settings:privacy-microphone') + await expect(openMicrophoneSettings('linux')).resolves.toBe(false) + }) + it('keeps live search suggestions behind the app origin and privacy preference', async () => { const { invoke } = collectHandlers() const handler = invoke.get('browser-agent:search-suggestions') diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 8d5861eb506..363b4bb88fc 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -19,14 +19,16 @@ import { isDesktopZoomPercent, isPendingDesktopScopeId, } from '@sim/desktop-bridge' +import { createLogger } from '@sim/logger' import { isTerminalOperation, isTerminalToolName, type TerminalToolArgs, } from '@sim/terminal-protocol' +import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import type { BrowserWindow, IpcMainEvent, IpcMainInvokeEvent, WebContents } from 'electron' -import { clipboard, ipcMain } from 'electron' +import { clipboard, ipcMain, shell } from 'electron' import { type BrowserToolQueueBoundary, cancelActiveTool, @@ -84,9 +86,35 @@ import type { ScopedEventRouter } from '@/main/scoped-event-router' import type { TerminalRegistry } from '@/main/terminal/registry' import { findCachedTerminalThemeProfile, listTerminalThemeProfiles } from '@/main/terminal-themes' +const logger = createLogger('DesktopIpc') + /** Workspace/chat ids are opaque tokens; anything else never reaches a URL. */ const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/ +const MICROPHONE_SETTINGS_URLS: Partial> = { + darwin: 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone', + win32: 'ms-settings:privacy-microphone', +} + +/** Opens the native microphone privacy pane without accepting a renderer-provided URL. */ +export async function openMicrophoneSettings( + platform: NodeJS.Platform = process.platform +): Promise { + const settingsUrl = MICROPHONE_SETTINGS_URLS[platform] + if (!settingsUrl) return false + + try { + await shell.openExternal(settingsUrl) + return true + } catch (error) { + logger.warn('Could not open microphone privacy settings', { + error: getErrorMessage(error), + platform, + }) + return false + } +} + /** * Desktop state is partitioned by the existing chat id. A new-chat view uses * the composer’s existing provisional key until the server assigns that id. @@ -627,6 +655,12 @@ export function registerIpcHandlers(deps: IpcDeps): void { handler: (url) => typeof url === 'string' ? openExternalSafe(url, deps.allowHttpLocalhost()) : false, }, + 'desktop:open-microphone-settings': { + kind: 'invoke', + gate: 'app-origin', + denied: false, + handler: () => openMicrophoneSettings(), + }, // OAuth connect handoff: the whole flow runs in the system browser (state // is cookie-bound to the initiating user agent), returning via loopback. 'desktop:oauth-connect': { diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 07d4e9e59e4..79e484ac924 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -114,6 +114,8 @@ function shellVersion(): string { const api: SimDesktopApi = { version: shellVersion(), openExternal: (url: string): Promise => ipcRenderer.invoke('desktop:open-external', url), + openMicrophoneSettings: (): Promise => + ipcRenderer.invoke('desktop:open-microphone-settings'), beginOAuthConnect: (providerId: string, scope?: DesktopOAuthConnectScope): Promise => ipcRenderer.invoke('desktop:oauth-connect', providerId, scope), onOAuthConnectComplete: (callback: (result: DesktopOAuthConnectResult) => void): (() => void) => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.test.tsx new file mode 100644 index 00000000000..8364c6d1fc1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.test.tsx @@ -0,0 +1,112 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { MicButton } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button' + +let container: HTMLDivElement +let root: Root +let animationFrameCallback: FrameRequestCallback | undefined +let nextAnimationFrameId: number + +function setReducedMotion(matches: boolean): void { + vi.stubGlobal( + 'matchMedia', + vi.fn(() => ({ + matches, + media: '(prefers-reduced-motion: reduce)', + onchange: null, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + addListener: vi.fn(), + removeListener: vi.fn(), + dispatchEvent: vi.fn(), + })) + ) +} + +function render(isListening: boolean, onToggle = vi.fn(), levels = new Float32Array(5)) { + act(() => { + root.render( + + ) + }) + return onToggle +} + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + animationFrameCallback = undefined + nextAnimationFrameId = 1 + setReducedMotion(false) + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + animationFrameCallback = callback + return nextAnimationFrameId++ + }) + ) + vi.stubGlobal('cancelAnimationFrame', vi.fn()) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.unstubAllGlobals() +}) + +describe('MicButton', () => { + it('keeps the waveform inside the circular active control', () => { + render(true) + + const button = container.querySelector('button') + const waveform = container.querySelector('svg[viewBox="0 0 18 18"]') + const bars = waveform?.querySelectorAll('line') + + expect(button?.className).toContain('size-[28px]') + expect(button?.className).toContain('overflow-hidden') + expect(button?.className).toContain('rounded-full') + expect(waveform?.classList.contains('size-[18px]')).toBe(true) + expect(bars).toHaveLength(5) + expect(Array.from(bars ?? []).map((bar) => bar.getAttribute('x1'))).toEqual([ + '3', + '6', + '9', + '12', + '15', + ]) + }) + + it('animates SVG attributes from the shared audio buffer without a React render', () => { + render(true, vi.fn(), new Float32Array([1, 1, 1, 1, 1])) + const firstBar = container.querySelector('line') + const initialY1 = firstBar?.getAttribute('y1') + + act(() => animationFrameCallback?.(16)) + + expect(firstBar?.getAttribute('y1')).not.toBe(initialY1) + expect(requestAnimationFrame).toHaveBeenCalledTimes(2) + }) + + it('exposes pressed state, toggles on click, and stops work for reduced motion', () => { + setReducedMotion(true) + const onToggle = render(true) + const button = container.querySelector('button') + + expect(button?.getAttribute('aria-pressed')).toBe('true') + expect(button?.getAttribute('aria-label')).toBe('Stop listening') + expect(requestAnimationFrame).not.toHaveBeenCalled() + + act(() => button?.click()) + expect(onToggle).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx index a4e25129823..8299e26cc27 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx @@ -1,30 +1,119 @@ 'use client' -import React from 'react' -import { cn, Mic, Tooltip } from '@sim/emcn' +import { memo, type RefObject, useEffect, useRef } from 'react' +import { Button, cn, Tooltip, usePrefersReducedMotion } from '@sim/emcn' +import { Mic } from '@sim/emcn/icons' + +const WAVEFORM_BAR_COUNT = 5 +const WAVEFORM_MIN_HEIGHT = 3 +const WAVEFORM_MAX_HEIGHT = 14 +const WAVEFORM_CENTER = 9 +const WAVEFORM_EASING = 0.24 interface MicButtonProps { isListening: boolean + audioLevelsRef: RefObject onToggle: () => void } -export const MicButton = React.memo(function MicButton({ isListening, onToggle }: MicButtonProps) { +interface VoiceWaveformProps { + audioLevelsRef: RefObject + isListening: boolean +} + +function VoiceWaveform({ audioLevelsRef, isListening }: VoiceWaveformProps) { + const prefersReducedMotion = usePrefersReducedMotion() + const barRefs = useRef>([]) + + useEffect(() => { + if (!isListening || prefersReducedMotion) return + + const heights = new Float32Array(WAVEFORM_BAR_COUNT).fill(WAVEFORM_MIN_HEIGHT) + let animationFrameId = 0 + + const draw = () => { + const levels = audioLevelsRef.current + + for (let index = 0; index < WAVEFORM_BAR_COUNT; index++) { + const level = levels?.[index] ?? 0 + const targetHeight = + WAVEFORM_MIN_HEIGHT + + Math.sqrt(Math.max(0, level)) * (WAVEFORM_MAX_HEIGHT - WAVEFORM_MIN_HEIGHT) + heights[index] += (targetHeight - heights[index]) * WAVEFORM_EASING + + const bar = barRefs.current[index] + if (!bar) continue + const halfHeight = heights[index] / 2 + bar.setAttribute('y1', String(WAVEFORM_CENTER - halfHeight)) + bar.setAttribute('y2', String(WAVEFORM_CENTER + halfHeight)) + } + + animationFrameId = window.requestAnimationFrame(draw) + } + + animationFrameId = window.requestAnimationFrame(draw) + return () => window.cancelAnimationFrame(animationFrameId) + }, [audioLevelsRef, isListening, prefersReducedMotion]) + + return ( + + {Array.from({ length: WAVEFORM_BAR_COUNT }, (_, index) => { + const x = 3 + index * 3 + return ( + { + barRefs.current[index] = element + }} + x1={x} + x2={x} + y1={WAVEFORM_CENTER - WAVEFORM_MIN_HEIGHT / 2} + y2={WAVEFORM_CENTER + WAVEFORM_MIN_HEIGHT / 2} + stroke='currentColor' + strokeLinecap='round' + strokeWidth='1.7' + /> + ) + })} + + ) +} + +export const MicButton = memo(function MicButton({ + isListening, + audioLevelsRef, + onToggle, +}: MicButtonProps) { return ( - + + + + + {isListening ? 'Stop listening' : 'Voice input'} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 1b2c6467340..7978e584e95 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -15,7 +15,7 @@ import { createLogger } from '@sim/logger' import { useParams } from 'next/navigation' import { getMothershipAttachmentPreviewUrl } from '@/lib/copilot/chat/attachment-preview' import { SIM_RESOURCE_DRAG_TYPE, SIM_RESOURCES_DRAG_TYPE } from '@/lib/copilot/resource-types' -import { isDesktopApp } from '@/lib/desktop' +import { getDesktopBridge } from '@/lib/desktop' import { MOTHERSHIP_ADD_CONTEXT_EVENT } from '@/lib/mothership/events' import { MOTHERSHIP_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useChatSurface } from '@/app/workspace/[workspaceId]/home/components/chat-surface-context' @@ -301,10 +301,19 @@ const UserInputImpl = forwardRef(function UserI function handleSpeechError(error: SpeechToTextError) { if (error === 'microphone-blocked') { + const desktopBridge = getDesktopBridge() toast.error( - isDesktopApp() + desktopBridge ? 'Microphone access is blocked. Allow Sim to use the microphone in your system privacy settings.' - : 'Microphone access is blocked. Allow it for this site and try again.' + : 'Microphone access is blocked. Allow it for this site and try again.', + desktopBridge?.openMicrophoneSettings + ? { + action: { + label: 'Open Settings', + onClick: () => void desktopBridge.openMicrophoneSettings?.(), + }, + } + : undefined ) return } @@ -316,6 +325,7 @@ const UserInputImpl = forwardRef(function UserI } const { + audioLevelsRef, isListening, isSupported: isSttSupported, toggleListening: rawToggle, @@ -635,7 +645,13 @@ const UserInputImpl = forwardRef(function UserI
- {isSttSupported && } + {isSttSupported && ( + + )} toggleListening: () => void resetTranscript: () => void } +const AUDIO_LEVEL_COUNT = 5 +const AUDIO_LEVEL_GAIN = 8 +const AUDIO_LEVEL_SMOOTHING = 0.55 + +function updateAudioLevels(input: Float32Array, levels: Float32Array): void { + const samplesPerLevel = Math.floor(input.length / levels.length) + + for (let levelIndex = 0; levelIndex < levels.length; levelIndex++) { + const start = levelIndex * samplesPerLevel + const end = levelIndex === levels.length - 1 ? input.length : start + samplesPerLevel + let sumOfSquares = 0 + + for (let sampleIndex = start; sampleIndex < end; sampleIndex++) { + const sample = input[sampleIndex] + sumOfSquares += sample * sample + } + + const rms = Math.sqrt(sumOfSquares / Math.max(1, end - start)) + const normalizedLevel = Math.min(1, rms * AUDIO_LEVEL_GAIN) + levels[levelIndex] = + levels[levelIndex] * AUDIO_LEVEL_SMOOTHING + normalizedLevel * (1 - AUDIO_LEVEL_SMOOTHING) + } +} + export function useSpeechToText({ onTranscript, onUsageLimitExceeded, @@ -90,6 +115,7 @@ export function useSpeechToText({ const streamRef = useRef(null) const audioContextRef = useRef(null) const processorRef = useRef(null) + const audioLevelsRef = useRef(new Float32Array(AUDIO_LEVEL_COUNT)) const pcmBufferRef = useRef([]) const sendIntervalRef = useRef | null>(null) @@ -175,6 +201,7 @@ export function useSpeechToText({ } pcmBufferRef.current = [] + audioLevelsRef.current.fill(0) isFirstChunkRef.current = true }, []) @@ -292,6 +319,7 @@ export function useSpeechToText({ processor.onaudioprocess = (e) => { const input = e.inputBuffer.getChannelData(0) + updateAudioLevels(input, audioLevelsRef.current) pcmBufferRef.current.push(new Float32Array(input)) } @@ -360,6 +388,8 @@ export function useSpeechToText({ streamRef.current = null } + audioLevelsRef.current.fill(0) + const wsToClose = wsRef.current wsRef.current = null if (wsToClose) { @@ -402,6 +432,7 @@ export function useSpeechToText({ return { isListening, isSupported, + audioLevelsRef, toggleListening, resetTranscript, } diff --git a/packages/desktop-bridge/contract-snapshot.ts b/packages/desktop-bridge/contract-snapshot.ts index 886076a0397..19a8bf2968c 100644 --- a/packages/desktop-bridge/contract-snapshot.ts +++ b/packages/desktop-bridge/contract-snapshot.ts @@ -193,8 +193,32 @@ export interface BrowserPageState { loading: boolean canGoBack: boolean canGoForward: boolean + /** Recoverable problem replacing the native page surface. Optional for older shells. */ + issue?: BrowserPageIssue } +/** A recoverable top-level page problem rendered by Sim instead of a blank native view. */ +export type BrowserPageIssue = + | { + kind: 'load-error' + /** Chromium network error number, such as -102 for connection refused. */ + code: number + /** Chromium network error name, such as ERR_CONNECTION_REFUSED. */ + description: string + /** The attempted URL, which may never have committed in WebContents. */ + url: string + } + | { + kind: 'crashed' + /** Chromium renderer exit reason, such as crashed or oom. */ + reason: string + url: string + } + | { + kind: 'unresponsive' + url: string + } + /** * One find-in-page request against the active tab. Backed by Chromium's own * find, so behaviour matches Chrome exactly — this only carries the query and @@ -237,6 +261,8 @@ export interface BrowserTabState { title: string loading: boolean active: boolean + /** Recoverable problem currently replacing this tab's native page surface. */ + issue?: BrowserPageIssue /** Pinned tabs are ordered before regular tabs and cannot be closed. */ pinned: boolean } @@ -1817,6 +1843,8 @@ export interface SimDesktopApi { /** Installed shell version (plain semver, e.g. `0.3.1`). */ version: string openExternal(url: string): Promise + /** Opens the operating system's microphone privacy settings when supported. */ + openMicrophoneSettings?(): Promise /** * Start the OAuth connect handoff for a provider: the whole flow runs in * the system browser and returns via loopback. Resolves false when the diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index 36b3f433c03..3c99fe52841 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -1045,6 +1045,8 @@ export interface SimDesktopApi { /** Installed shell version (plain semver, e.g. `0.3.1`). */ version: string openExternal(url: string): Promise + /** Opens the operating system's microphone privacy settings when supported. */ + openMicrophoneSettings?(): Promise /** * Start the OAuth connect handoff for a provider: the whole flow runs in * the system browser and returns via loopback. Resolves false when the From 90143992eb29a9a712aabb881bd76396224d8e40 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 26 Aug 2026 18:54:28 -0700 Subject: [PATCH 2/2] fix(desktop): hide unsupported microphone settings action --- apps/desktop/src/preload/index.test.ts | 17 +++++++++++++++++ apps/desktop/src/preload/index.ts | 8 ++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/preload/index.test.ts b/apps/desktop/src/preload/index.test.ts index 42cab747db4..4780e720e1e 100644 --- a/apps/desktop/src/preload/index.test.ts +++ b/apps/desktop/src/preload/index.test.ts @@ -45,4 +45,21 @@ describe('desktop preload bridge', () => { ['desktop:settings:set-browser-search-suggestions', false], ]) }) + + it('exposes native microphone settings only on supported platforms', async () => { + const exposed = exposeInMainWorld.mock.calls.find(([name]) => name === 'simDesktop')?.[1] as + | SimDesktopApi + | undefined + if (!exposed) throw new Error('Expected the desktop preload API to be exposed') + + const isSupportedPlatform = process.platform === 'darwin' || process.platform === 'win32' + expect(typeof exposed.openMicrophoneSettings).toBe( + isSupportedPlatform ? 'function' : 'undefined' + ) + + if (isSupportedPlatform) { + await exposed.openMicrophoneSettings?.() + expect(invoke).toHaveBeenLastCalledWith('desktop:open-microphone-settings') + } + }) }) diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 79e484ac924..d19949e7516 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -114,8 +114,12 @@ function shellVersion(): string { const api: SimDesktopApi = { version: shellVersion(), openExternal: (url: string): Promise => ipcRenderer.invoke('desktop:open-external', url), - openMicrophoneSettings: (): Promise => - ipcRenderer.invoke('desktop:open-microphone-settings'), + ...(process.platform === 'darwin' || process.platform === 'win32' + ? { + openMicrophoneSettings: (): Promise => + ipcRenderer.invoke('desktop:open-microphone-settings'), + } + : {}), beginOAuthConnect: (providerId: string, scope?: DesktopOAuthConnectScope): Promise => ipcRenderer.invoke('desktop:oauth-connect', providerId, scope), onOAuthConnectComplete: (callback: (result: DesktopOAuthConnectResult) => void): (() => void) => {