feat(chat): add live voice input waveform (#7146)

* feat(chat): add live voice input waveform

* fix(desktop): hide unsupported microphone settings action

Author: waleedlatif1

apps/desktop/src/main/ipc.test.ts

@@ -130,7 +130,7 @@ import {
 } from '@/main/browser-import'
 import { getSearchSuggestions } from '@/main/browser-search/suggestions'
 import { trackInputActivity } from '@/main/input-activity'
-import { type IpcDeps, registerIpcHandlers } from '@/main/ipc'
+import { type IpcDeps, openMicrophoneSettings, registerIpcHandlers } from '@/main/ipc'
 import { LocalFilesystemService } from '@/main/local-filesystem'
 import { TerminalRegistry } from '@/main/terminal/registry'
 import { findCachedTerminalThemeProfile, listTerminalThemeProfiles } from '@/main/terminal-themes'
@@ -337,6 +337,26 @@ describe('registerIpcHandlers', () => {
     expect(shell.openExternal).toHaveBeenCalledTimes(1)
   })
 
+  it('opens microphone privacy settings only for the trusted app origin', async () => {
+    const { invoke } = collectHandlers()
+    const handler = invoke.get('desktop:open-microphone-settings')
+
+    expect(await handler?.(evilEvent)).toBe(false)
+    expect(await handler?.(appEvent)).toBe(process.platform === 'darwin')
+    expect(shell.openExternal).toHaveBeenCalledTimes(process.platform === 'darwin' ? 1 : 0)
+  })
+
+  it('uses fixed native microphone settings URLs', async () => {
+    await expect(openMicrophoneSettings('darwin')).resolves.toBe(true)
+    expect(shell.openExternal).toHaveBeenLastCalledWith(
+      'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone'
+    )
+
+    await expect(openMicrophoneSettings('win32')).resolves.toBe(true)
+    expect(shell.openExternal).toHaveBeenLastCalledWith('ms-settings:privacy-microphone')
+    await expect(openMicrophoneSettings('linux')).resolves.toBe(false)
+  })
+
   it('keeps live search suggestions behind the app origin and privacy preference', async () => {
     const { invoke } = collectHandlers()
     const handler = invoke.get('browser-agent:search-suggestions')

apps/desktop/src/main/ipc.ts

@@ -19,14 +19,16 @@ import {
   isDesktopZoomPercent,
   isPendingDesktopScopeId,
 } from '@sim/desktop-bridge'
+import { createLogger } from '@sim/logger'
 import {
   isTerminalOperation,
   isTerminalToolName,
   type TerminalToolArgs,
 } from '@sim/terminal-protocol'
+import { getErrorMessage } from '@sim/utils/errors'
 import { isRecordLike } from '@sim/utils/object'
 import type { BrowserWindow, IpcMainEvent, IpcMainInvokeEvent, WebContents } from 'electron'
-import { clipboard, ipcMain } from 'electron'
+import { clipboard, ipcMain, shell } from 'electron'
 import {
   type BrowserToolQueueBoundary,
   cancelActiveTool,
@@ -84,9 +86,35 @@ import type { ScopedEventRouter } from '@/main/scoped-event-router'
 import type { TerminalRegistry } from '@/main/terminal/registry'
 import { findCachedTerminalThemeProfile, listTerminalThemeProfiles } from '@/main/terminal-themes'
 
+const logger = createLogger('DesktopIpc')
+
 /** Workspace/chat ids are opaque tokens; anything else never reaches a URL. */
 const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/
 
+const MICROPHONE_SETTINGS_URLS: Partial<Record<NodeJS.Platform, string>> = {
+  darwin: 'x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone',
+  win32: 'ms-settings:privacy-microphone',
+}
+
+/** Opens the native microphone privacy pane without accepting a renderer-provided URL. */
+export async function openMicrophoneSettings(
+  platform: NodeJS.Platform = process.platform
+): Promise<boolean> {
+  const settingsUrl = MICROPHONE_SETTINGS_URLS[platform]
+  if (!settingsUrl) return false
+
+  try {
+    await shell.openExternal(settingsUrl)
+    return true
+  } catch (error) {
+    logger.warn('Could not open microphone privacy settings', {
+      error: getErrorMessage(error),
+      platform,
+    })
+    return false
+  }
+}
+
 /**
  * Desktop state is partitioned by the existing chat id. A new-chat view uses
  * the composer’s existing provisional key until the server assigns that id.
@@ -627,6 +655,12 @@ export function registerIpcHandlers(deps: IpcDeps): void {
       handler: (url) =>
         typeof url === 'string' ? openExternalSafe(url, deps.allowHttpLocalhost()) : false,
     },
+    'desktop:open-microphone-settings': {
+      kind: 'invoke',
+      gate: 'app-origin',
+      denied: false,
+      handler: () => openMicrophoneSettings(),
+    },
     // OAuth connect handoff: the whole flow runs in the system browser (state
     // is cookie-bound to the initiating user agent), returning via loopback.
     'desktop:oauth-connect': {

apps/desktop/src/preload/index.test.ts

@@ -45,4 +45,21 @@ describe('desktop preload bridge', () => {
       ['desktop:settings:set-browser-search-suggestions', false],
     ])
   })
+
+  it('exposes native microphone settings only on supported platforms', async () => {
+    const exposed = exposeInMainWorld.mock.calls.find(([name]) => name === 'simDesktop')?.[1] as
+      | SimDesktopApi
+      | undefined
+    if (!exposed) throw new Error('Expected the desktop preload API to be exposed')
+
+    const isSupportedPlatform = process.platform === 'darwin' || process.platform === 'win32'
+    expect(typeof exposed.openMicrophoneSettings).toBe(
+      isSupportedPlatform ? 'function' : 'undefined'
+    )
+
+    if (isSupportedPlatform) {
+      await exposed.openMicrophoneSettings?.()
+      expect(invoke).toHaveBeenLastCalledWith('desktop:open-microphone-settings')
+    }
+  })
 })

apps/desktop/src/preload/index.ts

@@ -114,6 +114,12 @@ function shellVersion(): string {
 const api: SimDesktopApi = {
   version: shellVersion(),
   openExternal: (url: string): Promise<boolean> => ipcRenderer.invoke('desktop:open-external', url),
+  ...(process.platform === 'darwin' || process.platform === 'win32'
+    ? {
+        openMicrophoneSettings: (): Promise<boolean> =>
+          ipcRenderer.invoke('desktop:open-microphone-settings'),
+      }
+    : {}),
   beginOAuthConnect: (providerId: string, scope?: DesktopOAuthConnectScope): Promise<boolean> =>
     ipcRenderer.invoke('desktop:oauth-connect', providerId, scope),
   onOAuthConnectComplete: (callback: (result: DesktopOAuthConnectResult) => void): (() => void) => {

apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.test.tsx

@@ -0,0 +1,112 @@
+/**
+ * @vitest-environment jsdom
+ */
+import { act } from 'react'
+import { createRoot, type Root } from 'react-dom/client'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { MicButton } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button'
+
+let container: HTMLDivElement
+let root: Root
+let animationFrameCallback: FrameRequestCallback | undefined
+let nextAnimationFrameId: number
+
+function setReducedMotion(matches: boolean): void {
+  vi.stubGlobal(
+    'matchMedia',
+    vi.fn(() => ({
+      matches,
+      media: '(prefers-reduced-motion: reduce)',
+      onchange: null,
+      addEventListener: vi.fn(),
+      removeEventListener: vi.fn(),
+      addListener: vi.fn(),
+      removeListener: vi.fn(),
+      dispatchEvent: vi.fn(),
+    }))
+  )
+}
+
+function render(isListening: boolean, onToggle = vi.fn(), levels = new Float32Array(5)) {
+  act(() => {
+    root.render(
+      <MicButton
+        isListening={isListening}
+        audioLevelsRef={{ current: levels }}
+        onToggle={onToggle}
+      />
+    )
+  })
+  return onToggle
+}
+
+beforeEach(() => {
+  globalThis.IS_REACT_ACT_ENVIRONMENT = true
+  container = document.createElement('div')
+  document.body.appendChild(container)
+  root = createRoot(container)
+  animationFrameCallback = undefined
+  nextAnimationFrameId = 1
+  setReducedMotion(false)
+  vi.stubGlobal(
+    'requestAnimationFrame',
+    vi.fn((callback: FrameRequestCallback) => {
+      animationFrameCallback = callback
+      return nextAnimationFrameId++
+    })
+  )
+  vi.stubGlobal('cancelAnimationFrame', vi.fn())
+})
+
+afterEach(() => {
+  act(() => root.unmount())
+  container.remove()
+  vi.unstubAllGlobals()
+})
+
+describe('MicButton', () => {
+  it('keeps the waveform inside the circular active control', () => {
+    render(true)
+
+    const button = container.querySelector('button')
+    const waveform = container.querySelector('svg[viewBox="0 0 18 18"]')
+    const bars = waveform?.querySelectorAll('line')
+
+    expect(button?.className).toContain('size-[28px]')
+    expect(button?.className).toContain('overflow-hidden')
+    expect(button?.className).toContain('rounded-full')
+    expect(waveform?.classList.contains('size-[18px]')).toBe(true)
+    expect(bars).toHaveLength(5)
+    expect(Array.from(bars ?? []).map((bar) => bar.getAttribute('x1'))).toEqual([
+      '3',
+      '6',
+      '9',
+      '12',
+      '15',
+    ])
+  })
+
+  it('animates SVG attributes from the shared audio buffer without a React render', () => {
+    render(true, vi.fn(), new Float32Array([1, 1, 1, 1, 1]))
+    const firstBar = container.querySelector('line')
+    const initialY1 = firstBar?.getAttribute('y1')
+
+    act(() => animationFrameCallback?.(16))
+
+    expect(firstBar?.getAttribute('y1')).not.toBe(initialY1)
+    expect(requestAnimationFrame).toHaveBeenCalledTimes(2)
+  })
+
+  it('exposes pressed state, toggles on click, and stops work for reduced motion', () => {
+    setReducedMotion(true)
+    const onToggle = render(true)
+    const button = container.querySelector('button')
+
+    expect(button?.getAttribute('aria-pressed')).toBe('true')
+    expect(button?.getAttribute('aria-label')).toBe('Stop listening')
+    expect(requestAnimationFrame).not.toHaveBeenCalled()
+
+    act(() => button?.click())
+    expect(onToggle).toHaveBeenCalledOnce()
+  })
+})

apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mic-button/mic-button.tsx

@@ -1,30 +1,119 @@
 'use client'
 
-import React from 'react'
-import { cn, Mic, Tooltip } from '@sim/emcn'
+import { memo, type RefObject, useEffect, useRef } from 'react'
+import { Button, cn, Tooltip, usePrefersReducedMotion } from '@sim/emcn'
+import { Mic } from '@sim/emcn/icons'
+
+const WAVEFORM_BAR_COUNT = 5
+const WAVEFORM_MIN_HEIGHT = 3
+const WAVEFORM_MAX_HEIGHT = 14
+const WAVEFORM_CENTER = 9
+const WAVEFORM_EASING = 0.24
 
 interface MicButtonProps {
   isListening: boolean
+  audioLevelsRef: RefObject<Float32Array>
   onToggle: () => void
 }
 
-export const MicButton = React.memo(function MicButton({ isListening, onToggle }: MicButtonProps) {
+interface VoiceWaveformProps {
+  audioLevelsRef: RefObject<Float32Array>
+  isListening: boolean
+}
+
+function VoiceWaveform({ audioLevelsRef, isListening }: VoiceWaveformProps) {
+  const prefersReducedMotion = usePrefersReducedMotion()
+  const barRefs = useRef<Array<SVGLineElement | null>>([])
+
+  useEffect(() => {
+    if (!isListening || prefersReducedMotion) return
+
+    const heights = new Float32Array(WAVEFORM_BAR_COUNT).fill(WAVEFORM_MIN_HEIGHT)
+    let animationFrameId = 0
+
+    const draw = () => {
+      const levels = audioLevelsRef.current
+
+      for (let index = 0; index < WAVEFORM_BAR_COUNT; index++) {
+        const level = levels?.[index] ?? 0
+        const targetHeight =
+          WAVEFORM_MIN_HEIGHT +
+          Math.sqrt(Math.max(0, level)) * (WAVEFORM_MAX_HEIGHT - WAVEFORM_MIN_HEIGHT)
+        heights[index] += (targetHeight - heights[index]) * WAVEFORM_EASING
+
+        const bar = barRefs.current[index]
+        if (!bar) continue
+        const halfHeight = heights[index] / 2
+        bar.setAttribute('y1', String(WAVEFORM_CENTER - halfHeight))
+        bar.setAttribute('y2', String(WAVEFORM_CENTER + halfHeight))
+      }
+
+      animationFrameId = window.requestAnimationFrame(draw)
+    }
+
+    animationFrameId = window.requestAnimationFrame(draw)
+    return () => window.cancelAnimationFrame(animationFrameId)
+  }, [audioLevelsRef, isListening, prefersReducedMotion])
+
+  return (
+    <svg aria-hidden viewBox='0 0 18 18' className='size-[18px] overflow-hidden'>
+      {Array.from({ length: WAVEFORM_BAR_COUNT }, (_, index) => {
+        const x = 3 + index * 3
+        return (
+          <line
+            key={x}
+            ref={(element) => {
+              barRefs.current[index] = element
+            }}
+            x1={x}
+            x2={x}
+            y1={WAVEFORM_CENTER - WAVEFORM_MIN_HEIGHT / 2}
+            y2={WAVEFORM_CENTER + WAVEFORM_MIN_HEIGHT / 2}
+            stroke='currentColor'
+            strokeLinecap='round'
+            strokeWidth='1.7'
+          />
+        )
+      })}
+    </svg>
+  )
+}
+
+export const MicButton = memo(function MicButton({
+  isListening,
+  audioLevelsRef,
+  onToggle,
+}: MicButtonProps) {
   return (
     <Tooltip.Root>
       <Tooltip.Trigger asChild>
-        <button
+        <Button
           type='button'
+          variant={isListening ? 'active' : 'ghost'}
           onClick={onToggle}
           aria-label={isListening ? 'Stop listening' : 'Voice input'}
+          aria-pressed={isListening}
           className={cn(
-            'flex h-[28px] w-[28px] items-center justify-center rounded-full transition-colors',
-            isListening
-              ? 'bg-red-500 text-white hover:bg-red-600'
-              : 'text-[var(--text-icon)] hover:bg-[#F7F7F7] dark:hover:bg-[#303030]'
+            'relative size-[28px] overflow-hidden rounded-full p-0 transition-[background-color,color,scale] duration-150 ease-out active:scale-[0.96] motion-reduce:transition-none motion-reduce:active:scale-100',
+            !isListening &&
+              'text-[var(--text-icon)] hover-hover:bg-[var(--surface-hover)] hover-hover:text-[var(--text-icon)]'
           )}
         >
-          <Mic className='h-[16px] w-[16px]' />
-        </button>
+          <span
+            className={cn(
+              'absolute inset-0 flex items-center justify-center transition-[opacity,filter,scale] duration-300 [transition-timing-function:cubic-bezier(0.2,0,0,1)] motion-reduce:transition-none',
+              isListening ? 'scale-100 opacity-100 blur-0' : 'scale-[0.25] opacity-0 blur-[4px]'
+            )}
+          >
+            <VoiceWaveform audioLevelsRef={audioLevelsRef} isListening={isListening} />
+          </span>
+          <Mic
+            className={cn(
+              'size-[16px] transition-[opacity,filter,scale] duration-300 [transition-timing-function:cubic-bezier(0.2,0,0,1)] motion-reduce:transition-none',
+              isListening ? 'scale-[0.25] opacity-0 blur-[4px]' : 'scale-100 opacity-100 blur-0'
+            )}
+          />
+        </Button>
       </Tooltip.Trigger>
       <Tooltip.Content side='top'>{isListening ? 'Stop listening' : 'Voice input'}</Tooltip.Content>
     </Tooltip.Root>