From bc599455294044fc2db9021f827a4ca5c7c0b378 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 14:39:20 +0000 Subject: [PATCH 01/19] feat(voice): deliver provisional speech before transcription finishes Signed-off-by: John Matthew Tennant --- src-tauri/src/commands/native_voice.rs | 110 +++++++- .../__tests__/acpNotificationHandler.test.ts | 33 +++ src/features/chat/hooks/useChat.ts | 1 + src/features/chat/lib/backgroundSend.test.ts | 28 ++ src/features/chat/lib/backgroundSend.ts | 1 + src/features/chat/lib/sendCore.ts | 4 + src/features/chat/lib/steerCore.ts | 1 + src/features/chat/types.ts | 2 + .../api/voiceConversation.ts | 14 + .../useVoiceConversationController.test.ts | 227 +++++++++++++++++ .../hooks/useVoiceConversationController.ts | 241 +++++++++++++++++- .../lib/nativeAssistantSpeech.test.ts | 113 ++++++++ .../lib/nativeAssistantSpeech.ts | 16 ++ .../stores/voiceConversationStore.ts | 15 ++ src/shared/api/__tests__/acp.test.ts | 40 +++ src/shared/api/acp.ts | 23 +- 16 files changed, 847 insertions(+), 22 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 0ac09411d..e8db22999 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -205,6 +205,18 @@ pub(crate) enum NativeVoiceEvent { revision: u64, delivery_attempts: u8, }, + RecognitionPending { + session_id: String, + pending: bool, + revision: u64, + }, + InterimUser { + session_id: String, + lifecycle_id: String, + id: String, + text: String, + revision: u64, + }, Activity { session_id: String, activity: &'static str, @@ -1439,6 +1451,40 @@ fn reject_pending_transcript( Ok(TranscriptRejection { attempts, terminal }) } +struct EarlyVoiceTranscript { + utterance_id: String, + interim_text: String, + user_speaking: bool, + early_emitted: bool, +} + +impl Default for EarlyVoiceTranscript { + fn default() -> Self { + Self { + utterance_id: uuid::Uuid::new_v4().to_string(), + interim_text: String::new(), + user_speaking: false, + early_emitted: false, + } + } +} + +impl EarlyVoiceTranscript { + fn take_ready(&mut self) -> bool { + if self.user_speaking || self.early_emitted || self.interim_text.is_empty() { + return false; + } + self.early_emitted = true; + true + } + + fn reset(&mut self) { + self.utterance_id = uuid::Uuid::new_v4().to_string(); + self.interim_text.clear(); + self.early_emitted = false; + } +} + #[tauri::command] #[allow(clippy::too_many_arguments)] // Tauri injects four guards beside the lifecycle claim. pub async fn start_native_voice_conversation( @@ -1744,6 +1790,7 @@ pub async fn start_native_voice_conversation( .ok_or_else(|| "Voice admission was not initialized.".to_string())?; let event_state = state.inner().clone(); tauri::async_runtime::spawn(async move { + let mut early = EarlyVoiceTranscript::default(); while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { current.session_id.as_deref() == Some(session_id.as_str()) @@ -1758,7 +1805,20 @@ pub async fn start_native_voice_conversation( log::warn!("Voice input emitted duplicate readiness"); } berd_call::input::VoiceInputEvent::SpeakingChanged(speaking) => { + early.user_speaking = speaking; admission.set_user_speaking(speaking); + if early.take_ready() { + let _ = event_window.emit( + EVENT_NAME, + NativeVoiceEvent::InterimUser { + session_id: session_id.clone(), + lifecycle_id: lifecycle_id.clone(), + id: early.utterance_id.clone(), + text: early.interim_text.clone(), + revision, + }, + ); + } event_state.set_status_sound_input_activity(&session_id, revision, speaking); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), @@ -1774,11 +1834,33 @@ pub async fn start_native_voice_conversation( } berd_call::input::VoiceInputEvent::RecognitionPendingChanged(pending) => { admission.set_recognition_pending(pending); - // The runtime owns recognition-pending sequencing. Berd's - // renderer does not project that state yet. + if !pending { + early.reset(); + } + let _ = event_window.emit( + EVENT_NAME, + NativeVoiceEvent::RecognitionPending { + session_id: session_id.clone(), + pending, + revision, + }, + ); + } + berd_call::input::VoiceInputEvent::InterimTranscript(text) => { + early.interim_text = text.trim().to_string(); + if early.take_ready() { + let _ = event_window.emit( + EVENT_NAME, + NativeVoiceEvent::InterimUser { + session_id: session_id.clone(), + lifecycle_id: lifecycle_id.clone(), + id: early.utterance_id.clone(), + text: early.interim_text.clone(), + revision, + }, + ); + } } - // Berd's renderer does not show interim words yet. - berd_call::input::VoiceInputEvent::InterimTranscript(_) => {} berd_call::input::VoiceInputEvent::FinalTranscript { text, storage_receipt, @@ -1786,11 +1868,12 @@ pub async fn start_native_voice_conversation( let transcript = PendingTranscript { session_id: session_id.clone(), lifecycle_id: lifecycle_id.clone(), - id: uuid::Uuid::new_v4().to_string(), + id: early.utterance_id.clone(), text, revision, delivery_attempts: 0, }; + early.reset(); let Ok(disposition) = store_final_if_active( &runtime, &pending, @@ -2739,6 +2822,23 @@ mod tests { use super::*; use std::{thread, time::Instant}; + #[test] + fn recognition_resolution_allows_early_delivery_after_no_final_or_mute() { + let mut early = EarlyVoiceTranscript::default(); + early.user_speaking = true; + early.interim_text = "first words".into(); + assert!(!early.take_ready()); + early.user_speaking = false; + assert!(early.take_ready()); + assert!(!early.take_ready()); + let previous_id = early.utterance_id.clone(); + early.reset(); + assert_ne!(early.utterance_id, previous_id); + assert!(!early.take_ready()); + early.interim_text = "next words".into(); + assert!(early.take_ready()); + } + fn pending_transcript( session_id: &str, lifecycle_id: &str, diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index b5de9856a..38f8e0b75 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -2183,6 +2183,39 @@ describe("acpNotificationHandler", () => { }); }); + it.each([ + "provisional words", + "corrected words", + ])("replay exposes only the persistent final voice turn (%s)", async (finalText) => { + const sessionId = `early-voice-replay-${finalText}`; + useChatStore.setState({ loadingSessionIds: new Set([sessionId]) }); + for (const [messageId, text, hidden] of [ + ["provisional", "Provisional speech recognition.", true], + ["provisional", "provisional words", true], + ["final", "Final transcript, do not repeat an unchanged answer.", true], + ["final", finalText, false], + ] as const) { + await handleSessionNotification({ + sessionId, + update: { + sessionUpdate: "user_message_chunk", + messageId, + content: { + type: "text", + text, + ...(hidden ? { annotations: { audience: ["assistant"] } } : {}), + }, + }, + } as never); + } + expect(getReplayBuffer(sessionId)).toHaveLength(1); + expect(getReplayBuffer(sessionId)?.[0]).toMatchObject({ + id: "final", + role: "user", + content: [{ type: "text", text: finalText }], + }); + }); + it("replay restores skill chips from assistant-only user chunks", async () => { const replaySessionId = "replay-skill-session"; useChatStore.setState({ diff --git a/src/features/chat/hooks/useChat.ts b/src/features/chat/hooks/useChat.ts index 59da52eeb..e758cb764 100644 --- a/src/features/chat/hooks/useChat.ts +++ b/src/features/chat/hooks/useChat.ts @@ -206,6 +206,7 @@ export function useChat( persona: effectivePersonaInfo, attachments, assistantPrompt: sendOptions?.assistantPrompt, + assistantOnlyInput: sendOptions?.assistantOnlyInput, displayText: sendOptions?.displayText, userMessageId: sendOptions?.userMessageId, chips: sendOptions?.chips, diff --git a/src/features/chat/lib/backgroundSend.test.ts b/src/features/chat/lib/backgroundSend.test.ts index 6a4073a0d..4edd40511 100644 --- a/src/features/chat/lib/backgroundSend.test.ts +++ b/src/features/chat/lib/backgroundSend.test.ts @@ -43,6 +43,34 @@ describe("sendPromptInBackground", () => { ); }); + it("preserves assistant-only input through queued background dispatch", async () => { + await sendPromptInBackground( + "session-1", + "provisional words", + "goose", + undefined, + { + assistantOnlyInput: true, + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }, + ); + + expect(mocks.dispatchPrompt).toHaveBeenCalledWith( + "session-1", + "provisional words", + expect.objectContaining({ + assistantOnlyInput: true, + userMessageMetadata: { + origin: "voice_conversation", + userVisible: false, + }, + }), + ); + }); + it.each([ new QueuedSessionNotReadyError(), new QueuedMessageOwnershipLostError(), diff --git a/src/features/chat/lib/backgroundSend.ts b/src/features/chat/lib/backgroundSend.ts index fcd8e3077..71ba31691 100644 --- a/src/features/chat/lib/backgroundSend.ts +++ b/src/features/chat/lib/backgroundSend.ts @@ -44,6 +44,7 @@ export function sendPromptInBackground( : undefined, attachments, assistantPrompt: sendOptions.assistantPrompt, + assistantOnlyInput: sendOptions.assistantOnlyInput, displayText: sendOptions.displayText, chips: sendOptions.chips, userMessageMetadata: sendOptions.userMessageMetadata, diff --git a/src/features/chat/lib/sendCore.ts b/src/features/chat/lib/sendCore.ts index b2f5e4dde..fa834210a 100644 --- a/src/features/chat/lib/sendCore.ts +++ b/src/features/chat/lib/sendCore.ts @@ -55,6 +55,8 @@ export interface SendCoreOptions { attachments?: ChatAttachmentDraft[]; /** Assistant-audience prompt (skills/builder) merged ahead of the user text. */ assistantPrompt?: string; + /** Internal provisional context, persisted for the assistant only. */ + assistantOnlyInput?: boolean; /** Text shown in the transcript when it differs from the prompt sent. */ displayText?: string; /** Reuses a provisional local transcript row when the send commits. */ @@ -231,6 +233,7 @@ export async function dispatchPrompt( const tSendStart = performance.now(); const { assistantPrompt, + assistantOnlyInput, acpGooseMetadata, attachments, background, @@ -417,6 +420,7 @@ export async function dispatchPrompt( const promptPromise = acpSendMessage(sessionId, acpPrompt, { systemPrompt, ...(assistantPrompt ? { assistantPrompt } : {}), + assistantOnlyInput, personaId: persona?.id, personaName: persona?.name, goose: acpGooseMetadata, diff --git a/src/features/chat/lib/steerCore.ts b/src/features/chat/lib/steerCore.ts index c9a63ff90..3f182ba29 100644 --- a/src/features/chat/lib/steerCore.ts +++ b/src/features/chat/lib/steerCore.ts @@ -137,6 +137,7 @@ export async function steerPromptInSession( ...(sendOptions?.assistantPrompt ? { assistantPrompt: sendOptions.assistantPrompt } : {}), + assistantOnlyInput: sendOptions?.assistantOnlyInput, goose: sendOptions?.acpGooseMetadata, images: images?.map( (img) => [img.base64, img.mimeType] as [string, string], diff --git a/src/features/chat/types.ts b/src/features/chat/types.ts index de5b21cda..c411da8bc 100644 --- a/src/features/chat/types.ts +++ b/src/features/chat/types.ts @@ -73,6 +73,8 @@ export interface ChatSendOptions { /** Reuses a provisional local transcript row when the send commits. */ userMessageId?: string; assistantPrompt?: string; + /** Internal provisional context, persisted for the assistant only. */ + assistantOnlyInput?: boolean; chips?: MessageChip[]; userMessageMetadata?: Partial; acpGooseMetadata?: Record; diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index a80850283..86401bd85 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -227,6 +227,20 @@ export type VoiceConversationEvent = revision: number; deliveryAttempts: number; } + | { + type: "recognitionPending"; + sessionId: string; + pending: boolean; + revision: number; + } + | { + type: "interimUser"; + sessionId: string; + lifecycleId: string; + id: string; + text: string; + revision: number; + } | { type: "activity"; sessionId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 6752807b8..00a01ab98 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -92,6 +92,233 @@ function deferred() { } describe("voice transcript delivery coordination", () => { + it.each([ + "provisional words", + "Provisional words?", + "corrected words", + ])("reconciles early speech once (%s)", async (finalText) => { + const changed = finalText === "corrected words"; + const onSend = vi.fn(async (text, _persona, _attachments, options) => { + const state = useChatStore.getState(); + const existing = state.messagesBySession["session-1"]?.find( + (message) => message.id === options.userMessageId, + ); + const message = { + id: + existing?.id ?? + `row-${state.messagesBySession["session-1"]?.length ?? 0}`, + role: "user" as const, + created: 1, + content: [{ type: "text" as const, text }], + metadata: options.userMessageMetadata, + }; + if (existing) + state.updateMessage("session-1", existing.id, () => message); + else state.addMessage("session-1", message); + return true; + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + earlyTranscriptPending: null, + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + drainPendingTranscripts: vi.fn().mockResolvedValue(undefined), + }); + const { unmount } = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend, + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + const event = { + sessionId: "session-1", + lifecycleId: "lifecycle-1", + id: "early-test", + revision: 1, + text: "provisional words", + }; + await act(async () => { + await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); + }); + expect(onSend).toHaveBeenCalledTimes(1); + expect( + useVoiceConversationStore.getState().earlyTranscriptPending, + ).not.toBeNull(); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata + ?.userVisible, + ).toBe(false); + const final = { + ...event, + type: "user", + text: finalText, + deliveryAttempts: 0, + }; + if (changed) { + useChatStore.getState().setActiveRunId("session-1", "speculative-run"); + useChatStore.getState().setChatState("session-1", "streaming"); + let finalDelivery: void | Promise; + await act(async () => { + finalDelivery = voiceStoreMocks.subscriber?.(final); + await Promise.resolve(); + }); + expect(onSend).toHaveBeenCalledTimes(1); + // Old output arriving while the final is waiting retains provisional causality. + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata + ?.voiceUtteranceId, + ).toBe(`${event.id}:early`); + await act(async () => { + useChatStore.getState().setChatState("session-1", "idle"); + useChatStore.getState().settleActiveRun("session-1"); + await finalDelivery; + }); + } else { + await act(async () => { + await voiceStoreMocks.subscriber?.(final); + }); + } + await act(async () => { + await voiceStoreMocks.subscriber?.(final); + }); + expect(onSend).toHaveBeenCalledTimes(2); + expect(onSend.mock.calls[0]?.[3]).toMatchObject({ + assistantOnlyInput: true, + }); + expect(onSend.mock.calls[1]?.[3]).toMatchObject({ + assistantOnlyInput: false, + }); + const rows = useChatStore.getState().messagesBySession["session-1"] ?? []; + expect( + rows.filter((row) => row.metadata?.userVisible !== false), + ).toHaveLength(1); + expect(rows[1]).toMatchObject({ + content: [{ type: "text", text: final.text }], + metadata: { userVisible: true, voiceUtteranceId: event.id }, + }); + expect( + useVoiceConversationStore.getState().earlyTranscriptPending, + ).toBeNull(); + expect( + useVoiceConversationStore.getState().earlyTranscriptResolution?.accepted, + ).toBe(!changed); + unmount(); + }); + + it.each([ + false, + true, + ])("releases no-final provisional speech even when muted (%s)", async (muted) => { + const onSend = vi.fn(async (text, _persona, _attachments, options) => { + const state = useChatStore.getState(); + const existing = state.messagesBySession["session-1"]?.find( + (message) => message.id === options.userMessageId, + ); + const message = { + id: + existing?.id ?? + `row-${state.messagesBySession["session-1"]?.length ?? 0}`, + role: "user" as const, + created: 1, + content: [{ type: "text" as const, text }], + metadata: options.userMessageMetadata, + }; + if (existing) + state.updateMessage("session-1", existing.id, () => message); + else state.addMessage("session-1", message); + return true; + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 1, + }, + earlyTranscriptPending: null, + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + drainPendingTranscripts: vi.fn().mockResolvedValue(undefined), + }); + const { unmount } = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend, + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + const event = { + sessionId: "session-1", + lifecycleId: "lifecycle-1", + id: "abandoned-test", + revision: 1, + text: "provisional words", + }; + await act(async () => { + await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); + }); + expect(onSend).toHaveBeenCalledTimes(1); + expect( + useVoiceConversationStore.getState().earlyTranscriptPending, + ).not.toBeNull(); + expect( + useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata + ?.userVisible, + ).toBe(false); + + await act(async () => { + if (muted) useVoiceConversationStore.setState({ microphoneMuted: true }); + await voiceStoreMocks.subscriber?.({ + type: "recognitionPending", + sessionId: event.sessionId, + revision: event.revision, + pending: false, + }); + }); + expect( + useVoiceConversationStore.getState().earlyTranscriptPending, + ).toBeNull(); + expect( + useVoiceConversationStore.getState().earlyTranscriptResolution?.accepted, + ).toBe(false); + await act(async () => { + useVoiceConversationStore.setState({ microphoneMuted: false }); + await voiceStoreMocks.subscriber?.({ + ...event, + id: "next-utterance", + type: "interimUser", + }); + }); + expect(onSend).toHaveBeenCalledTimes(2); + await act(async () => { + await voiceStoreMocks.subscriber?.({ + type: "recognitionPending", + sessionId: event.sessionId, + revision: event.revision, + pending: false, + }); + }); + unmount(); + }); + it("suppresses floating controls only for the focused owner session", () => { const base = { activeSessionId: "session-1", diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 4a5cb37bb..7cb02a098 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -370,6 +370,42 @@ function voiceDeliveryOpportunity( return null; } +export function waitForVoiceRunSettlement(sessionId: string): Promise { + const settled = () => { + const runtime = useChatStore.getState().getSessionRuntime(sessionId); + return ( + runtime.activeRunId === null && + runtime.chatState === "idle" && + !runtime.isRunCancellationPending + ); + }; + if (settled()) return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + unsubscribeChat(); + unsubscribeVoice(); + }; + const unsubscribeChat = useChatStore.subscribe(() => { + if (!settled()) return; + cleanup(); + resolve(); + }); + const unsubscribeVoice = useVoiceConversationStore.subscribe((voice) => { + if ( + voice.status.lifecycle === "running" && + voice.status.sessionId === sessionId + ) + return; + cleanup(); + reject( + new VoiceTranscriptDeferredError( + "Voice transcript is waiting for its speculative run to settle.", + ), + ); + }); + }); +} + export function waitForVoiceDeliveryOpportunity( sessionId: string, ): Promise { @@ -524,11 +560,85 @@ export function resetVoiceUiWhenRunSettles( queueMicrotask(check); } +interface EarlyDelivery { + text: string; + key: string; + admitted: boolean; +} +const earlyDeliveries = new Map(); +const finalTranscriptsSeen = new Set(); + +function sameVoiceWords(first: string, second: string): boolean { + const words = (text: string) => + text + .toLowerCase() + .match(/[\p{L}\p{N}]+/gu) + ?.join(" ") ?? ""; + return words(first) === words(second); +} + +function voiceEventKey(event: { + sessionId: string; + lifecycleId: string; + revision: number; + id: string; +}): string { + return `${event.sessionId}\0${event.lifecycleId}\0${event.revision}\0${event.id}`; +} + +function earlyEventIsCurrent( + event: { sessionId: string; revision: number }, + requireMicrophone = true, +) { + const { status, microphoneMuted } = useVoiceConversationStore.getState(); + return ( + status.lifecycle === "running" && + status.sessionId === event.sessionId && + status.revision === event.revision && + status.ownerWindowLabel === getCurrentWindow().label && + (!requireMicrophone || !microphoneMuted) + ); +} + function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; observeChainedVoiceStatus(); - subscribeToVoiceConversationEvents(async (event) => { + useVoiceConversationStore.subscribe((state, previous) => { + if ( + state.microphoneMuted && + !previous.microphoneMuted && + state.earlyTranscriptPending + ) { + const early = earlyDeliveries.get(state.earlyTranscriptPending); + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: early + ? { + key: early.key, + finalKey: state.earlyTranscriptPending, + accepted: false, + } + : null, + }); + } + if ( + state.status.revision === previous.status.revision && + state.status.sessionId === previous.status.sessionId && + state.status.lifecycle === previous.status.lifecycle + ) + return; + earlyDeliveries.clear(); + finalTranscriptsSeen.clear(); + if (state.earlyTranscriptPending !== null) + useVoiceConversationStore.setState({ earlyTranscriptPending: null }); + }); + subscribeToVoiceConversationEvents(async (incoming) => { + const isEarly = incoming.type === "interimUser"; + const event = + incoming.type === "interimUser" + ? { ...incoming, type: "user" as const, deliveryAttempts: 0 } + : incoming; if (event.type === "cleanShutdown" || event.type === "controlsDismissed") { return; } @@ -542,6 +652,24 @@ function ensureVoiceEventDeliveryInitialized() { addErrorNotification(sessionId ?? null, event.message); return; } + if (event.type === "recognitionPending") { + if (event.pending) return; + if (!earlyEventIsCurrent(event, false)) return; + const pending = + useVoiceConversationStore.getState().earlyTranscriptPending; + // A final already queued owns reconciliation. Otherwise recognition has + // ended without a final. Release the gate even if admission is waiting. + if (!pending || finalTranscriptsSeen.has(pending)) return; + const early = earlyDeliveries.get(pending); + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: early + ? { key: early.key, finalKey: pending, accepted: false } + : null, + }); + earlyDeliveries.delete(pending); + return; + } if (event.type === "activity") return; if (event.type !== "user" || !event.text.trim()) return; if ( @@ -555,9 +683,38 @@ function ensureVoiceEventDeliveryInitialized() { return; } + const eventKey = voiceEventKey(event); + if (!isEarly) { + // Bounded by lifecycle, and retained only while an early delivery exists. + if (earlyDeliveries.has(eventKey)) finalTranscriptsSeen.add(eventKey); + } const deliveryRevision = event.revision; const shouldNotifyFailure = event.deliveryAttempts === 0; + if (isEarly) { + if ( + !earlyEventIsCurrent(event) || + earlyDeliveries.has(eventKey) || + useVoiceConversationStore.getState().earlyTranscriptPending !== null + ) + return; + earlyDeliveries.set(eventKey, { + text: event.text, + key: voiceEventKey({ ...event, id: `${event.id}:early` }), + admitted: false, + }); + useVoiceConversationStore.setState({ earlyTranscriptPending: eventKey }); + } return enqueueVoiceTranscriptDelivery(event.sessionId, async () => { + const early = earlyDeliveries.get(eventKey); + if ( + isEarly && + (!early || + !earlyEventIsCurrent(event) || + finalTranscriptsSeen.has(eventKey)) + ) { + return; + } + if (isEarly && activeRouteIsBlocked(event.sessionId)) return; if (activeRouteIsBlocked(event.sessionId)) { throw new VoiceTranscriptDeferredError( "Voice transcript is waiting for its bound chat to become available.", @@ -566,6 +723,7 @@ function ensureVoiceEventDeliveryInitialized() { const route = activeSendRoute; if (!route || route.sessionId !== event.sessionId) { + if (isEarly) return; const message = "Voice transcript could not be sent because its bound chat is unavailable."; if (shouldNotifyFailure) addErrorNotification(event.sessionId, message); @@ -573,28 +731,53 @@ function ensureVoiceEventDeliveryInitialized() { } const store = useVoiceConversationStore.getState(); - store.setUiState("user-speaking"); - + if (!isEarly) store.setUiState("user-speaking"); + const releaseEarly = (accepted: boolean) => { + if (!early) return; + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: { + key: early.key, + finalKey: eventKey, + accepted, + }, + }); + earlyDeliveries.delete(eventKey); + finalTranscriptsSeen.delete(eventKey); + }; const sendOptions = { + assistantOnlyInput: isEarly, userMessageMetadata: { + ...(isEarly ? { userVisible: false } : { userVisible: true }), origin: "voice_conversation" as const, - voiceUtteranceId: event.id, + voiceUtteranceId: isEarly ? `${event.id}:early` : event.id, voiceConversationLifecycleId: event.lifecycleId, voiceConversationRevision: event.revision, }, acpGooseMetadata: { origin: "voice_conversation", - voiceUtteranceId: event.id, + voiceUtteranceId: isEarly ? `${event.id}:early` : event.id, voiceConversationLifecycleId: event.lifecycleId, voiceConversationRevision: event.revision, }, }; try { + // Keep the provisional row and its causal identity immutable. A changed + // final must not become the cause of late messages from the old run. + if ( + !isEarly && + early?.admitted && + !sameVoiceWords(early.text, event.text) + ) + await waitForVoiceRunSettlement(event.sessionId); // This runs inside the per-session queue, so a prior send can change // the opportunity to steer before the next transcript is evaluated. - const opportunity = await waitForVoiceDeliveryOpportunity( - event.sessionId, - ); + const opportunity = isEarly + ? voiceDeliveryOpportunity(event.sessionId) + : await waitForVoiceDeliveryOpportunity(event.sessionId); + // Provisional delivery is best effort. Never block retained finals or + // the next utterance waiting for a run that has not started yet. + if (!opportunity) return; if (activeRouteIsBlocked(event.sessionId)) { throw new VoiceTranscriptDeferredError( "Voice transcript is waiting for its bound chat to become available.", @@ -606,13 +789,31 @@ function ensureVoiceEventDeliveryInitialized() { "Voice transcript could not be sent because its bound chat is unavailable.", ); } + if ( + isEarly && + (!early || + !earlyEventIsCurrent(event) || + finalTranscriptsSeen.has(eventKey)) + ) + return; const playbackNotice = takeVoicePlaybackNotices(event.sessionId); const displayOptions = { ...sendOptions, - ...(playbackNotice ? { assistantPrompt: playbackNotice } : {}), + assistantPrompt: [ + playbackNotice, + isEarly + ? "The user paused. This is provisional speech recognition and may be corrected. Do not treat it as confirmation for irreversible actions." + : early?.admitted + ? sameVoiceWords(early.text, event.text) + ? "This is the final transcript of the provisional speech already sent. It is unchanged. Persist this user turn, but do not repeat an answer already given or in progress unless the request changed." + : "Speech recognition correction: the final user transcript replaces the provisional words in the prior voice turn. Discard any answer based on the provisional wording and answer the corrected request." + : null, + ] + .filter(Boolean) + .join("\n\n"), displayText: event.text, }; - store.setUiState("agent-working"); + if (!isEarly) store.setUiState("agent-working"); const delivered = opportunity === "steer" ? await steerPromptInSession( @@ -631,9 +832,9 @@ function ensureVoiceEventDeliveryInitialized() { if (delivered === false) { // Runtime state may have changed between the opportunity read and // send admission. Re-evaluate once, then steer if a run now exists. - const retryOpportunity = await waitForVoiceDeliveryOpportunity( - event.sessionId, - ); + const retryOpportunity = isEarly + ? voiceDeliveryOpportunity(event.sessionId) + : await waitForVoiceDeliveryOpportunity(event.sessionId); if (retryOpportunity !== "steer") { throw new Error( "Voice transcript was not accepted by the chat session.", @@ -647,10 +848,24 @@ function ensureVoiceEventDeliveryInitialized() { { throwOnError: true }, ); } + if (isEarly && early) early.admitted = true; + if (!isEarly) + releaseEarly( + Boolean(early?.admitted && sameVoiceWords(early.text, event.text)), + ); // Admission releases the transcript queue so later speech can steer. // UI working state follows the actual chat runtime independently. resetVoiceUiWhenRunSettles(event.sessionId, deliveryRevision); } catch (deliveryError) { + if (isEarly) { + // Early delivery is best effort. Retained final delivery remains the + // authoritative recovery path. Keep the speech gate until then. + console.warn( + "Provisional voice delivery was not admitted", + deliveryError, + ); + return; + } const current = useVoiceConversationStore.getState(); if (deliveryError instanceof VoiceTranscriptDeferredError) { if ( diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 6385aee21..fed6e2cda 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -271,6 +271,8 @@ describe("native assistant speech stream", () => { assistantSpeaking: false, microphoneMuted: false, latestFinalizedTranscriptKey: null, + earlyTranscriptPending: null, + earlyTranscriptResolution: null, }); }); @@ -505,6 +507,117 @@ describe("native assistant speech stream", () => { ).toMatchObject({ text: source }); }); + it.each([ + false, + true, + ])("holds speculative output and resolves its final (accepted: %s)", async (accepted) => { + const earlyKey = ["session-1", "lifecycle-1", "1", "early:early"].join( + "\0", + ); + const finalKey = ["session-1", "lifecycle-1", "1", "early"].join("\0"); + startNativeAssistantSpeech("session-1", vi.fn()); + useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); + useChatStore + .getState() + .setMessages("session-1", [ + voiceUser("early:early", { userVisible: false }), + assistant( + [{ type: "text", text: "Speculative answer." }], + "completed", + "early-answer", + ), + ]); + await new Promise((resolve) => window.setTimeout(resolve, 15)); + expect(mocks.start).not.toHaveBeenCalled(); + finalizeVoiceTranscript("early"); + await new Promise((resolve) => window.setTimeout(resolve, 15)); + expect(mocks.start).not.toHaveBeenCalled(); + // The provisional row is immutable. A late message from that same run + // arrives before changed-final dispatch is allowed to begin. + useChatStore + .getState() + .addMessage( + "session-1", + assistant( + [{ type: "text", text: "Late speculative answer." }], + "completed", + "late-early-answer", + ), + ); + useChatStore.getState().addMessage("session-1", voiceUser("early")); + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: { key: earlyKey, finalKey, accepted }, + }); + if (accepted) { + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + expect.any(String), + "Speculative answer.", + ), + ); + } else { + await new Promise((resolve) => window.setTimeout(resolve, 15)); + expect(mocks.start).not.toHaveBeenCalled(); + useChatStore + .getState() + .addMessage( + "session-1", + assistant( + [{ type: "text", text: "Corrected answer." }], + "completed", + "corrected-answer", + ), + ); + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + expect.any(String), + "Corrected answer.", + ), + ); + expect( + mocks.append.mock.calls.every( + ([, text]) => !text.toLowerCase().includes("speculative"), + ), + ).toBe(true); + } + }); + + it("releases an early reply after a real speaking-to-pause-to-final sequence", async () => { + const earlyKey = ["session-1", "lifecycle-1", "1", "paused:early"].join( + "\0", + ); + const finalKey = ["session-1", "lifecycle-1", "1", "paused"].join("\0"); + startNativeAssistantSpeech("session-1", vi.fn()); + useVoiceConversationStore.setState({ userSpeaking: true }); + useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); + useChatStore + .getState() + .setMessages("session-1", [ + voiceUser("paused:early", { userVisible: false }), + assistant( + [{ type: "text", text: "Early paused answer." }], + "completed", + "paused-answer", + ), + ]); + useVoiceConversationStore.setState({ userSpeaking: false }); + await new Promise((resolve) => window.setTimeout(resolve, 300)); + expect(mocks.start).not.toHaveBeenCalled(); + finalizeVoiceTranscript("paused"); + useChatStore.getState().addMessage("session-1", voiceUser("paused")); + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: { key: earlyKey, finalKey, accepted: true }, + }); + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + expect.any(String), + "Early paused answer.", + ), + ); + }); + it("pushes raw assistant deltas without frontend sentence segmentation", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 546ba03d5..e6f296175 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1432,6 +1432,18 @@ export function startNativeAssistantSpeech( return; } const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + cacheCausalTranscriptKeys(messages); + const resolution = voice.earlyTranscriptResolution; + if (resolution?.accepted) { + for (const [id, key] of causalTranscriptKeyByMessage) { + if (key === resolution.key) + causalTranscriptKeyByMessage.set(id, resolution.finalKey); + } + } + // Do not consume or invalidate speculative text while recognition or its + // final delivery is unresolved. The controller releases this gate only + // after the final row and the agent's correction have been admitted. + if (voice.earlyTranscriptPending !== null) return; if ( heldSpeech || voice.userSpeaking || @@ -1792,6 +1804,10 @@ export function startNativeAssistantSpeech( inspect(); return; } + if (voice.earlyTranscriptPending !== null) { + inspect(); + return; + } if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) { if (heldReleaseTimer !== null) { window.clearTimeout(heldReleaseTimer); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 380d765dd..1cbffe7b3 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -58,6 +58,14 @@ interface VoiceConversationStore { hydrated: boolean; requestedStartSessionId: string | null; latestFinalizedTranscriptKey: string | null; + // Internal speech gate, not a visual state. Speculative replies cannot be + // spoken until their final transcript has been reconciled. + earlyTranscriptPending: string | null; + earlyTranscriptResolution: { + key: string; + finalKey: string; + accepted: boolean; + } | null; init: () => Promise; refreshStatus: () => Promise; requestStart: (sessionId: string) => void; @@ -311,6 +319,8 @@ export const useVoiceConversationStore = create( hydrated: false, requestedStartSessionId: null, latestFinalizedTranscriptKey: null, + earlyTranscriptPending: null, + earlyTranscriptResolution: null, requestStart: (sessionId) => set({ requestedStartSessionId: sessionId }), clearRequestedStart: (sessionId) => @@ -441,6 +451,8 @@ export const useVoiceConversationStore = create( revision: event.revision, }, uiState: "listening", + earlyTranscriptPending: null, + earlyTranscriptResolution: null, microphoneMuted: false, latestFinalizedTranscriptKey: null, error: null, @@ -463,6 +475,9 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } + case "recognitionPending": + case "interimUser": + return state; case "user": return { ...state, diff --git a/src/shared/api/__tests__/acp.test.ts b/src/shared/api/__tests__/acp.test.ts index 9b645d0d1..d0013d482 100644 --- a/src/shared/api/__tests__/acp.test.ts +++ b/src/shared/api/__tests__/acp.test.ts @@ -177,6 +177,46 @@ describe("acpSteerMessage", () => { }); describe("acpSendMessage", () => { + it.each([ + false, + true, + ])("uses real assistant-only audience for provisional input (steer: %s)", async (steer) => { + const registry = await import("../acpSessionRegistry"); + const { acpSendMessage, acpSteerMessage } = await import("../acp"); + registry.registerPreparedSession( + "early-audience", + "goose", + "/tmp/project", + "test-model", + ); + const options = { + assistantOnlyInput: true, + assistantPrompt: "Provisional speech.", + }; + if (steer) + await acpSteerMessage("early-audience", "run-1", "early words", options); + else await acpSendMessage("early-audience", "early words", options); + const provisional = (steer ? mockSteerSession : mockPrompt).mock.calls.at( + -1, + )?.[1]; + expect(provisional).toEqual([ + { + type: "text", + text: "Provisional speech.", + annotations: { audience: ["assistant"] }, + }, + { + type: "text", + text: "early words", + annotations: { audience: ["assistant"] }, + }, + ]); + if (steer) await acpSteerMessage("early-audience", "run-1", "final words"); + else await acpSendMessage("early-audience", "final words"); + expect( + (steer ? mockSteerSession : mockPrompt).mock.calls.at(-1)?.[1], + ).toEqual([{ type: "text", text: "final words" }]); + }); beforeEach(() => { vi.clearAllMocks(); vi.resetModules(); diff --git a/src/shared/api/acp.ts b/src/shared/api/acp.ts index c7c8c156f..3e9d0b313 100644 --- a/src/shared/api/acp.ts +++ b/src/shared/api/acp.ts @@ -55,6 +55,8 @@ export interface AcpProvider { export interface AcpSendMessageOptions { systemPrompt?: string; assistantPrompt?: string; + /** Internal provisional context, persisted for the assistant only. */ + assistantOnlyInput?: boolean; personaId?: string; personaName?: string; goose?: Record; @@ -166,6 +168,7 @@ async function acpSendMessageNow( const { systemPrompt, assistantPrompt, + assistantOnlyInput, personaId, personaName, goose, @@ -246,7 +249,13 @@ async function acpSendMessageNow( annotations: { audience: ["assistant"] }, }); } - content.push({ type: "text", text: prompt }); + content.push({ + type: "text", + text: prompt, + ...(assistantOnlyInput + ? { annotations: { audience: ["assistant" as const] } } + : {}), + }); if (images) { for (const [data, mimeType] of images) { content.push({ type: "image", data, mimeType } as ContentBlock); @@ -302,11 +311,11 @@ export async function acpSteerMessage( prompt: string, options: Pick< AcpSendMessageOptions, - "assistantPrompt" | "goose" | "images" + "assistantPrompt" | "assistantOnlyInput" | "goose" | "images" > = {}, ): Promise { sessionRegistry.requireSessionInvocationSelection(sessionId); - const { assistantPrompt, goose, images } = options; + const { assistantPrompt, assistantOnlyInput, goose, images } = options; const content: ContentBlock[] = []; const assistantText = assistantPrompt?.trim(); if (assistantText) { @@ -316,7 +325,13 @@ export async function acpSteerMessage( annotations: { audience: ["assistant"] }, }); } - content.push({ type: "text", text: prompt }); + content.push({ + type: "text", + text: prompt, + ...(assistantOnlyInput + ? { annotations: { audience: ["assistant" as const] } } + : {}), + }); if (images) { for (const [data, mimeType] of images) { content.push({ type: "image", data, mimeType } as ContentBlock); From 8d94889db9fa6b22ff75313ad427710da502ade1 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 14:45:23 +0000 Subject: [PATCH 02/19] test(voice): exercise provisional and final call delivery in the rendered app Signed-off-by: John Matthew Tennant --- tests/e2e/fixtures/tauri-mock.ts | 65 +++++++ tests/e2e/voice-conversation.spec.ts | 258 ++++++++++++++++++++++++++- 2 files changed, 319 insertions(+), 4 deletions(-) diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index 8f61e93e9..0675c5d0c 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -24,6 +24,7 @@ export function buildInitScript(options?: { realAcpUrl?: string; homeDir?: string; modelId?: string; + holdAcpPrompts?: boolean; personas?: unknown[]; skills?: unknown[]; projects?: unknown[]; @@ -120,6 +121,9 @@ export function buildInitScript(options?: { return ` (() => { + const HOLD_ACP_PROMPTS = ${options?.holdAcpPrompts ?? false}; + const ACP_REQUESTS = []; + const PENDING_ACP_PROMPTS = new Map(); const PERSONAS = ${personas}; const SKILLS = ${skills}; const PROJECTS = ${projects}; @@ -313,8 +317,23 @@ export function buildInitScript(options?: { })(); const SKILL_SOURCES = SKILLS.map(skillToSourceEntry); const POCKET_VOICE_SPOKEN_TEXTS = []; + const VOICE_STREAM_TEXTS = new Map(); + let voiceSpeechId = 0; window.__GOOSE_E2E__ = { + acpRequests: () => clone(ACP_REQUESTS), + completeAcpPrompts: () => { + for (const [id, { socket, sessionId }] of PENDING_ACP_PROMPTS) { + socket.dispatchEvent(new MessageEvent("message", { + data: JSON.stringify(jsonRpcResult(id, { stopReason: "end_turn" })), + })); + window.__GOOSE_E2E__.emitAcpNotification({ + sessionId, + update: { sessionUpdate: "session_info_update", _meta: { goose: { activeRunId: null } } }, + }); + } + PENDING_ACP_PROMPTS.clear(); + }, listAgentSources: () => clone(AGENT_SOURCES), clearAgentSources: () => { AGENT_SOURCES = PERSONAS.map(personaToSourceEntry); @@ -410,6 +429,8 @@ export function buildInitScript(options?: { _meta: { messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, + providerId: session.providerId, + modelId: session.modelId, }, cwd: session.workingDir, })), @@ -476,6 +497,11 @@ export function buildInitScript(options?: { } return jsonRpcResult(message.id, {}); } + case "_goose/unstable/session/steer": + return jsonRpcResult(message.id, { + runId: message.params.expectedRunId, + messageId: "mock-steer-" + message.id, + }); case "session/prompt": { const session = findSession(message.params?.sessionId); if (session) { @@ -530,6 +556,10 @@ export function buildInitScript(options?: { isConfigured: entry.configured, })), }); + case "_goose/unstable/providers/secrets/list": + return jsonRpcResult(message.id, { secrets: [{ provider: "openai", name: "OPENAI_API_KEY", storage: "secret_store", status: "unknown", configured: true, hasSecret: true, canDelete: true, canConfigure: false }] }); + case "_goose/unstable/defaults/read": + case "_goose/unstable/defaults/save": case "_goose/defaults/read": case "_goose/defaults/save": return jsonRpcResult(message.id, { @@ -673,6 +703,11 @@ export function buildInitScript(options?: { send(raw) { const message = JSON.parse(raw); + ACP_REQUESTS.push(clone(message)); + if (HOLD_ACP_PROMPTS && message.method === "session/prompt") { + PENDING_ACP_PROMPTS.set(message.id, { socket: this, sessionId: message.params.sessionId }); + return; + } const response = message && typeof message === "object" && "id" in message ? handleAcpRequest(message) @@ -802,6 +837,32 @@ export function buildInitScript(options?: { ); emitTauriEvent("openai-voice:settings-changed", null); return Promise.resolve(null); + case "prepare_native_voice_assistant_speech": + return Promise.resolve({ outcome: "admitted", speechId: ++voiceSpeechId }); + case "start_pocket_voice_stream": + case "start_siri_voice_stream": + VOICE_STREAM_TEXTS.set(args.streamId, ""); + return Promise.resolve(true); + case "append_pocket_voice_stream": + case "append_siri_voice_stream": { + VOICE_STREAM_TEXTS.set(args.streamId, (VOICE_STREAM_TEXTS.get(args.streamId) ?? "") + args.text); + return Promise.resolve(null); + } + case "flush_pocket_voice_stream": + case "flush_siri_voice_stream": + case "finish_pocket_voice_stream": + case "finish_siri_voice_stream": { + const text = VOICE_STREAM_TEXTS.get(args.streamId); + if (text) { + POCKET_VOICE_SPOKEN_TEXTS.push(text); + VOICE_STREAM_TEXTS.set(args.streamId, ""); + } + if (cmd.startsWith("finish")) queueMicrotask(() => emitTauriEvent( + cmd.includes("siri") ? "siri-voice:stream-event" : "pocket-voice:stream-event", + { streamId: args.streamId, state: "completed", error: null }, + )); + return Promise.resolve(null); + } case "speak_pocket_voice": POCKET_VOICE_SPOKEN_TEXTS.push(args?.text); return new Promise((resolve) => @@ -876,6 +937,8 @@ export function buildInitScript(options?: { }); // ---- Sessions / Misc ---- + case "list_berd_app_skills": + return Promise.resolve({ skills: [] }); case "get_session_window_support": return Promise.resolve({ supported: false, @@ -897,6 +960,8 @@ export function buildInitScript(options?: { updatedAt: session.updatedAt, messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, + providerId: session.providerId, + modelId: session.modelId, conversationBefore: session.conversationBefore, })), ); diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 264aa0f74..62e096588 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -2,6 +2,8 @@ import { test as base, expect, type Page } from "@playwright/test"; import { buildInitScript } from "./fixtures/tauri-mock"; +base.use({ screenshot: "off", video: "off", trace: "off" }); + const SESSION_ID = "voice-session"; const ASSISTANT_MESSAGE_ID = "assistant-message"; const SPOKEN_TEXT = "This playback-confirmed reply should be visible."; @@ -34,7 +36,7 @@ const test = base.extend<{ voice: Page }>({ test("queues assistant text through native playback", async ({ voice: page, -}, testInfo) => { +}) => { await page.goto("/"); await page.getByText("Voice playback", { exact: true }).click(); @@ -111,8 +113,256 @@ test("queues assistant text through native playback", async ({ ), ) .toEqual([SPOKEN_TEXT]); - await testInfo.attach("playback-confirmed-text-status", { - body: await page.screenshot(), - contentType: "image/png", +}); + +// These exercise the rendered controller and ACP adapter, not native recognition +// or a real model. Prompt completion is held so pause/final races are deterministic. +type AcpRequest = { + method: string; + params?: { + prompt?: { text?: string; annotations?: { audience?: string[] } }[]; + _meta?: { goose?: { voiceUtteranceId?: string } }; + }; +}; +type VoiceHarness = { + emitTauriEvent: (event: string, payload: unknown) => void; + emitAcpNotification: (params: unknown) => void; + acpRequests: () => AcpRequest[]; + completeAcpPrompts: () => void; + pocketVoiceSpokenTexts: () => string[]; +}; + +async function nativeEvent(page: Page, payload: object) { + await page.evaluate( + (payload) => { + const harness = ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__; + harness.emitTauriEvent("voice-conversation:event", payload); + }, + { sessionId: SESSION_ID, revision: 2, ...payload }, + ); +} + +async function requests(page: Page) { + return page.evaluate(() => + (window as typeof window & { __GOOSE_E2E__: VoiceHarness }).__GOOSE_E2E__ + .acpRequests() + .filter((request) => + ["session/prompt", "_goose/unstable/session/steer"].includes( + request.method, + ), + ), + ); +} + +async function spoken(page: Page) { + return page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.pocketVoiceSpokenTexts(), + ); +} + +async function completePrompts(page: Page) { + await page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.completeAcpPrompts(), + ); +} + +async function assistantOutput(page: Page, text: string, runId: string) { + await page.evaluate( + ({ text, runId, sessionId }) => { + const harness = ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__; + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "session_info_update", + _meta: { goose: { activeRunId: runId } }, + }, + }); + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + messageId: `message-${runId}`, + content: { type: "text", text }, + }, + }); + // A tool boundary flushes the speech sentence without completing the run. + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: `tool-${runId}`, + title: "read_file", + rawInput: { path: "/tmp/voice-e2e.txt" }, + }, + }); + }, + { text, runId, sessionId: SESSION_ID }, + ); +} + +const utterance = { + lifecycleId: "early-delivery-lifecycle", + id: "utterance-1", + text: "Please explain the pause delivery behavior.", +}; + +base.describe("early voice delivery", () => { + base.beforeEach(async ({ page }) => { + await page.addInitScript({ + content: buildInitScript({ + holdAcpPrompts: true, + sessions: [ + { + sessionId: SESSION_ID, + title: "Early voice", + messageCount: 1, + providerId: "openai", + modelId: "gpt-4.1", + }, + ], + voiceConversationStatus: { + available: true, + unavailableReason: null, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: 1, + }, + enabledExperiments: ["voice-conversation"], + }), + }); + await page.goto("/"); + await page.getByText("Early voice", { exact: true }).click(); + await expect(page.getByRole("textbox").first()).toBeVisible(); + await page.evaluate( + () => + new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ), + ); + await nativeEvent(page, { + type: "startup", + ownerWindowLabel: "main", + line: "Ready", + }); + await nativeEvent(page, { type: "interimUser", ...utterance }); + await expect.poll(() => requests(page)).toHaveLength(1); + const early = (await requests(page))[0]; + expect(early.method).toBe("session/prompt"); + expect( + early.params?.prompt?.find((block) => block.text === utterance.text), + ).toMatchObject({ annotations: { audience: ["assistant"] } }); + await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( + 0, + ); }); + + base( + "unchanged final is visible once and releases withheld assistant speech", + async ({ page }) => { + const answer = "The unchanged provisional answer can now be spoken."; + await assistantOutput(page, answer, "early-run"); + await expect.poll(() => spoken(page)).toEqual([]); + await nativeEvent(page, { + type: "user", + ...utterance, + deliveryAttempts: 0, + }); + await expect.poll(() => requests(page)).toHaveLength(2); + expect((await requests(page))[1].method).toBe( + "_goose/unstable/session/steer", + ); + expect( + (await requests(page))[1].params?.prompt?.find( + (block) => block.text === utterance.text, + )?.annotations, + ).toBeUndefined(); + await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( + 1, + ); + await expect.poll(() => spoken(page)).toEqual([answer]); + await nativeEvent(page, { + type: "user", + ...utterance, + deliveryAttempts: 0, + }); + await completePrompts(page); + await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( + 1, + ); + expect(await requests(page)).toHaveLength(2); + expect(await spoken(page)).toEqual([answer]); + }, + ); + + base( + "changed final never speaks the old answer, including late old output", + async ({ page }) => { + const oldAnswer = + "This obsolete provisional answer must never be spoken."; + const finalText = + "Actually explain the corrected final delivery behavior."; + await assistantOutput(page, oldAnswer, "obsolete-run"); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + await assistantOutput( + page, + " Late obsolete output must also stay unspoken.", + "obsolete-run", + ); + expect(await requests(page)).toHaveLength(1); + expect(await spoken(page)).toEqual([]); + await completePrompts(page); + await expect.poll(() => requests(page)).toHaveLength(2); + expect((await requests(page))[1].method).toBe("session/prompt"); + await expect(page.getByText(finalText, { exact: true })).toHaveCount(1); + await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( + 0, + ); + const correctedAnswer = + "Only the corrected final answer should be spoken."; + await assistantOutput(page, correctedAnswer, "corrected-run"); + await completePrompts(page); + await expect.poll(() => spoken(page)).toEqual([correctedAnswer]); + }, + ); + + base( + "recognition ending without a final discards speculative speech and permits the next utterance", + async ({ page }) => { + await assistantOutput( + page, + "An abandoned answer must remain unspoken.", + "abandoned-run", + ); + await nativeEvent(page, { type: "recognitionPending", pending: false }); + await completePrompts(page); + await nativeEvent(page, { + type: "interimUser", + ...utterance, + id: "utterance-2", + text: "The next utterance.", + }); + await expect.poll(() => requests(page)).toHaveLength(2); + expect(await spoken(page)).toEqual([]); + await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( + 0, + ); + await nativeEvent(page, { type: "cleanShutdown" }); + await completePrompts(page); + expect(await spoken(page)).toEqual([]); + }, + ); }); From 2be1baa7c8b2d2c4323a600744ab246fb65689d7 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 15:04:01 +0000 Subject: [PATCH 03/19] fix(voice): reconcile recognition while early speech is held Signed-off-by: John Matthew Tennant --- .../lib/nativeAssistantSpeech.test.ts | 69 +++++++++++++++++++ .../lib/nativeAssistantSpeech.ts | 16 +++-- tests/e2e/fixtures/tauri-mock.ts | 1 + tests/e2e/voice-conversation.spec.ts | 52 ++++++++++++-- 4 files changed, 126 insertions(+), 12 deletions(-) diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index fed6e2cda..3d305cda4 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -618,6 +618,75 @@ describe("native assistant speech stream", () => { ); }); + it("resolves interrupted playback recognition while early speech is gated", async () => { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Interrupted reply." }]), + ]); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + emit("started"); + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); + emit("interrupted"); + + const finalKey = ["session-1", "lifecycle-1", "1", "interrupt"].join("\0"); + const earlyKey = `${finalKey}:early`; + useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); + useVoiceConversationStore.setState({ userSpeaking: false }); + useChatStore + .getState() + .addMessage( + "session-1", + voiceUser("interrupt:early", { userVisible: false }), + ); + useChatStore + .getState() + .addMessage( + "session-1", + assistant( + [{ type: "text", text: "Reply to the interruption." }], + "completed", + "interruption-answer", + ), + ); + finalizeVoiceTranscript("interrupt"); + useChatStore.getState().addMessage("session-1", voiceUser("interrupt")); + expect(mocks.start).toHaveBeenCalledTimes(1); + useVoiceConversationStore.setState({ + earlyTranscriptPending: null, + earlyTranscriptResolution: { key: earlyKey, finalKey, accepted: true }, + }); + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + expect.any(String), + "Reply to the interruption.", + ), + ); + mocks.streamHandler?.({ + streamId: mocks.start.mock.calls[1]?.[0] as string, + state: "completed", + error: null, + }); + useChatStore + .getState() + .addMessage( + "session-1", + assistant( + [{ type: "text", text: "Later replies still speak." }], + "completed", + "later-answer", + ), + ); + await vi.waitFor(() => + expect(mocks.append).toHaveBeenCalledWith( + expect.any(String), + "Later replies still speak.", + ), + ); + }); + it("pushes raw assistant deltas without frontend sentence segmentation", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index e6f296175..258978ad7 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1804,11 +1804,11 @@ export function startNativeAssistantSpeech( inspect(); return; } - if (voice.earlyTranscriptPending !== null) { - inspect(); - return; - } - if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) { + if ( + finalizedTranscriptChanged && + !pendingUserRecognitionSegment && + voice.earlyTranscriptPending === null + ) { if (heldReleaseTimer !== null) { window.clearTimeout(heldReleaseTimer); heldReleaseTimer = null; @@ -1837,6 +1837,12 @@ export function startNativeAssistantSpeech( return; } if (finalizedTranscriptChanged && pendingUserRecognitionSegment) { + if (voice.earlyTranscriptPending !== null) { + resolvePendingRecognitionSegment(false); + discardResumableInterruption(); + inspect(); + return; + } holdAssistantChanges( useChatStore.getState().messagesBySession[sessionId] ?? [], ); diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index 0675c5d0c..72f9aac05 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -355,6 +355,7 @@ export function buildInitScript(options?: { }, emitTauriEvent, pocketVoiceSpokenTexts: () => clone(POCKET_VOICE_SPOKEN_TEXTS), + pocketVoiceStreamIds: () => [...VOICE_STREAM_TEXTS.keys()], }; function nowIso() { diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 62e096588..f5df1479f 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -130,6 +130,7 @@ type VoiceHarness = { acpRequests: () => AcpRequest[]; completeAcpPrompts: () => void; pocketVoiceSpokenTexts: () => string[]; + pocketVoiceStreamIds: () => string[]; }; async function nativeEvent(page: Page, payload: object) { @@ -253,7 +254,35 @@ base.describe("early voice delivery", () => { ownerWindowLabel: "main", line: "Ready", }); + await assistantOutput(page, "The reply being interrupted.", "prior-run"); + await expect + .poll(() => spoken(page)) + .toEqual(["The reply being interrupted."]); + await nativeEvent(page, { type: "activity", activity: "user-speaking" }); + await page.evaluate(() => { + const harness = ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__; + harness.emitTauriEvent("pocket-voice:stream-event", { + streamId: harness.pocketVoiceStreamIds()[0], + state: "interrupted", + error: null, + delivery: { segments: [] }, + }); + }); + await page.evaluate((sessionId) => { + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "session_info_update", + _meta: { goose: { activeRunId: null } }, + }, + }); + }, SESSION_ID); await nativeEvent(page, { type: "interimUser", ...utterance }); + await nativeEvent(page, { type: "activity", activity: "user-idle" }); await expect.poll(() => requests(page)).toHaveLength(1); const early = (await requests(page))[0]; expect(early.method).toBe("session/prompt"); @@ -270,7 +299,9 @@ base.describe("early voice delivery", () => { async ({ page }) => { const answer = "The unchanged provisional answer can now be spoken."; await assistantOutput(page, answer, "early-run"); - await expect.poll(() => spoken(page)).toEqual([]); + await expect + .poll(() => spoken(page)) + .toEqual(["The reply being interrupted."]); await nativeEvent(page, { type: "user", ...utterance, @@ -288,7 +319,9 @@ base.describe("early voice delivery", () => { await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( 1, ); - await expect.poll(() => spoken(page)).toEqual([answer]); + await expect + .poll(() => spoken(page)) + .toEqual(["The reply being interrupted.", answer]); await nativeEvent(page, { type: "user", ...utterance, @@ -299,7 +332,10 @@ base.describe("early voice delivery", () => { 1, ); expect(await requests(page)).toHaveLength(2); - expect(await spoken(page)).toEqual([answer]); + expect(await spoken(page)).toEqual([ + "The reply being interrupted.", + answer, + ]); }, ); @@ -323,7 +359,7 @@ base.describe("early voice delivery", () => { "obsolete-run", ); expect(await requests(page)).toHaveLength(1); - expect(await spoken(page)).toEqual([]); + expect(await spoken(page)).toEqual(["The reply being interrupted."]); await completePrompts(page); await expect.poll(() => requests(page)).toHaveLength(2); expect((await requests(page))[1].method).toBe("session/prompt"); @@ -335,7 +371,9 @@ base.describe("early voice delivery", () => { "Only the corrected final answer should be spoken."; await assistantOutput(page, correctedAnswer, "corrected-run"); await completePrompts(page); - await expect.poll(() => spoken(page)).toEqual([correctedAnswer]); + await expect + .poll(() => spoken(page)) + .toEqual(["The reply being interrupted.", correctedAnswer]); }, ); @@ -356,13 +394,13 @@ base.describe("early voice delivery", () => { text: "The next utterance.", }); await expect.poll(() => requests(page)).toHaveLength(2); - expect(await spoken(page)).toEqual([]); + expect(await spoken(page)).toEqual(["The reply being interrupted."]); await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( 0, ); await nativeEvent(page, { type: "cleanShutdown" }); await completePrompts(page); - expect(await spoken(page)).toEqual([]); + expect(await spoken(page)).toEqual(["The reply being interrupted."]); }, ); }); From c9c6d10f4f2ff0dc023c16388198dde9315eb272 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 15:09:50 +0000 Subject: [PATCH 04/19] fix(voice): release speculative speech after terminal delivery failure Signed-off-by: John Matthew Tennant --- .../hooks/useVoiceConversationController.ts | 8 ++++ .../stores/voiceConversationStore.test.ts | 47 +++++++++++++++++++ .../stores/voiceConversationStore.ts | 18 +++++-- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 7cb02a098..c4f035c6f 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -605,6 +605,14 @@ function ensureVoiceEventDeliveryInitialized() { deliveryInitialized = true; observeChainedVoiceStatus(); useVoiceConversationStore.subscribe((state, previous) => { + if ( + previous.earlyTranscriptPending !== null && + state.earlyTranscriptPending === null && + state.earlyTranscriptResolution?.accepted === false + ) { + earlyDeliveries.delete(previous.earlyTranscriptPending); + finalTranscriptsSeen.delete(previous.earlyTranscriptPending); + } if ( state.microphoneMuted && !previous.microphoneMuted && diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 1096a190b..4fbe73445 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -267,6 +267,53 @@ describe("voice conversation store lifecycle ordering", () => { unsubscribe(); }); + it("holds speculative speech during retries but rejects it on terminal delivery failure", async () => { + const module = await import("./voiceConversationStore"); + const unsubscribe = module.subscribeToVoiceConversationEvents(() => + Promise.reject(new Error("chat unavailable")), + ); + await module.useVoiceConversationStore.getState().init(); + const transcript = { + sessionId: "session-1", + lifecycleId: "lifecycle-1", + revision: 1, + id: "failed-final", + text: "Cannot deliver", + deliveryAttempts: 0, + }; + const key = ["session-1", "lifecycle-1", "1", "failed-final"].join("\0"); + module.useVoiceConversationStore.setState({ + earlyTranscriptPending: key, + earlyTranscriptResolution: null, + latestFinalizedTranscriptKey: "prior-transcript", + }); + mocks.drain.mockResolvedValue([transcript]); + await expect( + module.useVoiceConversationStore + .getState() + .drainPendingTranscripts("session-1"), + ).rejects.toThrow("delivery was rejected"); + expect( + module.useVoiceConversationStore.getState().earlyTranscriptPending, + ).toBe(key); + mocks.reject.mockResolvedValueOnce({ attempts: 3, terminal: true }); + await expect( + module.useVoiceConversationStore + .getState() + .drainPendingTranscripts("session-1"), + ).rejects.toThrow("delivery was rejected"); + expect(module.useVoiceConversationStore.getState()).toMatchObject({ + earlyTranscriptPending: null, + earlyTranscriptResolution: { + key: `${key}:early`, + finalKey: key, + accepted: false, + }, + latestFinalizedTranscriptKey: "prior-transcript", + }); + unsubscribe(); + }); + it("removes terminally rejected transcripts from pending rollback chains", async () => { mocks.reject.mockResolvedValue({ attempts: 3, terminal: true }); const firstDelivery = deferred(); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 1cbffe7b3..b62b4df87 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -241,11 +241,21 @@ async function deliverTranscriptOnce( priorFinalizedTranscriptKeys.set(pendingKey, priorKey); } } - useVoiceConversationStore.setState((state) => - state.latestFinalizedTranscriptKey === finalizedKey + useVoiceConversationStore.setState((state) => ({ + ...(state.latestFinalizedTranscriptKey === finalizedKey ? { latestFinalizedTranscriptKey: priorKey } - : state, - ); + : {}), + ...(state.earlyTranscriptPending === finalizedKey + ? { + earlyTranscriptPending: null, + earlyTranscriptResolution: { + key: `${finalizedKey}:early`, + finalKey: finalizedKey, + accepted: false, + }, + } + : {}), + })); } } return accepted ? "accepted" : deferred ? "deferred" : "rejected"; From aa9217a93e96a8b6b2f6edb12b7c837b4b74dcfb Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 15:09:50 +0000 Subject: [PATCH 05/19] fix(voice): preserve meaningful signs in final transcript comparison Signed-off-by: John Matthew Tennant --- .../hooks/useVoiceConversationController.test.ts | 16 ++++++++++++---- .../hooks/useVoiceConversationController.ts | 5 +++-- tests/e2e/voice-conversation.spec.ts | 5 ++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 00a01ab98..a506c8deb 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -96,8 +96,12 @@ describe("voice transcript delivery coordination", () => { "provisional words", "Provisional words?", "corrected words", + "set the temperature to -10 degrees", + "calculate 3 + 4", ])("reconciles early speech once (%s)", async (finalText) => { - const changed = finalText === "corrected words"; + const changed = !["provisional words", "Provisional words?"].includes( + finalText, + ); const onSend = vi.fn(async (text, _persona, _attachments, options) => { const state = useChatStore.getState(); const existing = state.messagesBySession["session-1"]?.find( @@ -145,9 +149,13 @@ describe("voice transcript delivery coordination", () => { const event = { sessionId: "session-1", lifecycleId: "lifecycle-1", - id: "early-test", + id: `early-test-${finalText}`, revision: 1, - text: "provisional words", + text: finalText.includes("temperature") + ? "set the temperature to 10 degrees" + : finalText.includes("calculate") + ? "calculate 3 - 4" + : "provisional words", }; await act(async () => { await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); @@ -166,7 +174,7 @@ describe("voice transcript delivery coordination", () => { text: finalText, deliveryAttempts: 0, }; - if (changed) { + if (finalText === "corrected words") { useChatStore.getState().setActiveRunId("session-1", "speculative-run"); useChatStore.getState().setChatState("session-1", "streaming"); let finalDelivery: void | Promise; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index c4f035c6f..d955249b4 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -572,8 +572,9 @@ function sameVoiceWords(first: string, second: string): boolean { const words = (text: string) => text .toLowerCase() - .match(/[\p{L}\p{N}]+/gu) - ?.join(" ") ?? ""; + .trim() + .replace(/[.!?]+$/u, "") + .replace(/\s+/gu, " "); return words(first) === words(second); } diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index f5df1479f..9a01e6ec9 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -212,7 +212,7 @@ async function assistantOutput(page: Page, text: string, runId: string) { const utterance = { lifecycleId: "early-delivery-lifecycle", id: "utterance-1", - text: "Please explain the pause delivery behavior.", + text: "Set the temperature to 10 degrees.", }; base.describe("early voice delivery", () => { @@ -344,8 +344,7 @@ base.describe("early voice delivery", () => { async ({ page }) => { const oldAnswer = "This obsolete provisional answer must never be spoken."; - const finalText = - "Actually explain the corrected final delivery behavior."; + const finalText = "Set the temperature to -10 degrees."; await assistantOutput(page, oldAnswer, "obsolete-run"); await nativeEvent(page, { type: "user", From 93e289fc3dfcd9ddb7d22207a723cf01ed4e5d40 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 15:17:07 +0000 Subject: [PATCH 06/19] fix(voice): retain provisional run causality across microphone mute Signed-off-by: John Matthew Tennant --- .../hooks/useVoiceConversationController.test.ts | 12 +++++++++++- .../hooks/useVoiceConversationController.ts | 10 +++------- .../stores/voiceConversationStore.ts | 5 ++++- tests/e2e/voice-conversation.spec.ts | 1 + 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index a506c8deb..f2f05534d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -96,6 +96,7 @@ describe("voice transcript delivery coordination", () => { "provisional words", "Provisional words?", "corrected words", + "corrected words after mute", "set the temperature to -10 degrees", "calculate 3 + 4", ])("reconciles early speech once (%s)", async (finalText) => { @@ -174,7 +175,15 @@ describe("voice transcript delivery coordination", () => { text: finalText, deliveryAttempts: 0, }; - if (finalText === "corrected words") { + if (finalText === "corrected words after mute") { + await act(async () => { + useVoiceConversationStore.setState({ microphoneMuted: true }); + }); + expect( + useVoiceConversationStore.getState().earlyTranscriptPending, + ).toBeNull(); + } + if (finalText.startsWith("corrected words")) { useChatStore.getState().setActiveRunId("session-1", "speculative-run"); useChatStore.getState().setChatState("session-1", "streaming"); let finalDelivery: void | Promise; @@ -222,6 +231,7 @@ describe("voice transcript delivery coordination", () => { expect( useVoiceConversationStore.getState().earlyTranscriptResolution?.accepted, ).toBe(!changed); + act(() => useVoiceConversationStore.setState({ microphoneMuted: false })); unmount(); }); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index d955249b4..5e18a152a 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -606,13 +606,9 @@ function ensureVoiceEventDeliveryInitialized() { deliveryInitialized = true; observeChainedVoiceStatus(); useVoiceConversationStore.subscribe((state, previous) => { - if ( - previous.earlyTranscriptPending !== null && - state.earlyTranscriptPending === null && - state.earlyTranscriptResolution?.accepted === false - ) { - earlyDeliveries.delete(previous.earlyTranscriptPending); - finalTranscriptsSeen.delete(previous.earlyTranscriptPending); + if (state.earlyTranscriptResolution?.terminal === true) { + earlyDeliveries.delete(state.earlyTranscriptResolution.finalKey); + finalTranscriptsSeen.delete(state.earlyTranscriptResolution.finalKey); } if ( state.microphoneMuted && diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index b62b4df87..dced41a7b 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -65,6 +65,7 @@ interface VoiceConversationStore { key: string; finalKey: string; accepted: boolean; + terminal?: boolean; } | null; init: () => Promise; refreshStatus: () => Promise; @@ -245,13 +246,15 @@ async function deliverTranscriptOnce( ...(state.latestFinalizedTranscriptKey === finalizedKey ? { latestFinalizedTranscriptKey: priorKey } : {}), - ...(state.earlyTranscriptPending === finalizedKey + ...(state.earlyTranscriptPending === finalizedKey || + state.earlyTranscriptResolution?.finalKey === finalizedKey ? { earlyTranscriptPending: null, earlyTranscriptResolution: { key: `${finalizedKey}:early`, finalKey: finalizedKey, accepted: false, + terminal: true, }, } : {}), diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 9a01e6ec9..3a5ca7557 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -346,6 +346,7 @@ base.describe("early voice delivery", () => { "This obsolete provisional answer must never be spoken."; const finalText = "Set the temperature to -10 degrees."; await assistantOutput(page, oldAnswer, "obsolete-run"); + await nativeEvent(page, { type: "microphoneMute", muted: true }); await nativeEvent(page, { type: "user", ...utterance, From 986a7f79497e17c210decada09a10a896e96c5b9 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 15:21:40 +0000 Subject: [PATCH 07/19] fix(voice): keep provisional answers held through microphone mute Signed-off-by: John Matthew Tennant --- .../useVoiceConversationController.test.ts | 18 ++++++++++++------ .../hooks/useVoiceConversationController.ts | 17 ----------------- .../lib/nativeAssistantSpeech.test.ts | 1 + tests/e2e/voice-conversation.spec.ts | 1 + 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index f2f05534d..10cd936c4 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -95,14 +95,17 @@ describe("voice transcript delivery coordination", () => { it.each([ "provisional words", "Provisional words?", + "provisional words after mute", "corrected words", "corrected words after mute", "set the temperature to -10 degrees", "calculate 3 + 4", ])("reconciles early speech once (%s)", async (finalText) => { - const changed = !["provisional words", "Provisional words?"].includes( - finalText, - ); + const changed = ![ + "provisional words", + "Provisional words?", + "provisional words after mute", + ].includes(finalText); const onSend = vi.fn(async (text, _persona, _attachments, options) => { const state = useChatStore.getState(); const existing = state.messagesBySession["session-1"]?.find( @@ -172,16 +175,19 @@ describe("voice transcript delivery coordination", () => { const final = { ...event, type: "user", - text: finalText, + text: + finalText === "provisional words after mute" + ? "provisional words" + : finalText, deliveryAttempts: 0, }; - if (finalText === "corrected words after mute") { + if (finalText.endsWith("after mute")) { await act(async () => { useVoiceConversationStore.setState({ microphoneMuted: true }); }); expect( useVoiceConversationStore.getState().earlyTranscriptPending, - ).toBeNull(); + ).not.toBeNull(); } if (finalText.startsWith("corrected words")) { useChatStore.getState().setActiveRunId("session-1", "speculative-run"); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 5e18a152a..7560df7b0 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -610,23 +610,6 @@ function ensureVoiceEventDeliveryInitialized() { earlyDeliveries.delete(state.earlyTranscriptResolution.finalKey); finalTranscriptsSeen.delete(state.earlyTranscriptResolution.finalKey); } - if ( - state.microphoneMuted && - !previous.microphoneMuted && - state.earlyTranscriptPending - ) { - const early = earlyDeliveries.get(state.earlyTranscriptPending); - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: early - ? { - key: early.key, - finalKey: state.earlyTranscriptPending, - accepted: false, - } - : null, - }); - } if ( state.status.revision === previous.status.revision && state.status.sessionId === previous.status.sessionId && diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 3d305cda4..d044b37c5 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -529,6 +529,7 @@ describe("native assistant speech stream", () => { ]); await new Promise((resolve) => window.setTimeout(resolve, 15)); expect(mocks.start).not.toHaveBeenCalled(); + useVoiceConversationStore.setState({ microphoneMuted: true }); finalizeVoiceTranscript("early"); await new Promise((resolve) => window.setTimeout(resolve, 15)); expect(mocks.start).not.toHaveBeenCalled(); diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 3a5ca7557..4b4f698c9 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -299,6 +299,7 @@ base.describe("early voice delivery", () => { async ({ page }) => { const answer = "The unchanged provisional answer can now be spoken."; await assistantOutput(page, answer, "early-run"); + await nativeEvent(page, { type: "microphoneMute", muted: true }); await expect .poll(() => spoken(page)) .toEqual(["The reply being interrupted."]); From 10853ed9906d3c616bb30ed90f7d6d1ef6919ce3 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 13:05:16 -0400 Subject: [PATCH 08/19] fix(voice): preserve case-sensitive transcript corrections Signed-off-by: John Matthew Tennant --- .../useVoiceConversationController.test.ts | 9 ++- .../hooks/useVoiceConversationController.ts | 1 - tests/e2e/voice-conversation.spec.ts | 77 ++++++++++--------- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 10cd936c4..517544e14 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -94,16 +94,17 @@ function deferred() { describe("voice transcript delivery coordination", () => { it.each([ "provisional words", - "Provisional words?", + "provisional words?", "provisional words after mute", "corrected words", "corrected words after mute", "set the temperature to -10 degrees", "calculate 3 + 4", + "open foo.ts", ])("reconciles early speech once (%s)", async (finalText) => { const changed = ![ "provisional words", - "Provisional words?", + "provisional words?", "provisional words after mute", ].includes(finalText); const onSend = vi.fn(async (text, _persona, _attachments, options) => { @@ -159,7 +160,9 @@ describe("voice transcript delivery coordination", () => { ? "set the temperature to 10 degrees" : finalText.includes("calculate") ? "calculate 3 - 4" - : "provisional words", + : finalText === "open foo.ts" + ? "open Foo.ts" + : "provisional words", }; await act(async () => { await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 7560df7b0..ace373834 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -571,7 +571,6 @@ const finalTranscriptsSeen = new Set(); function sameVoiceWords(first: string, second: string): boolean { const words = (text: string) => text - .toLowerCase() .trim() .replace(/[.!?]+$/u, "") .replace(/\s+/gu, " "); diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 4b4f698c9..c6cc321ae 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -340,43 +340,46 @@ base.describe("early voice delivery", () => { }, ); - base( - "changed final never speaks the old answer, including late old output", - async ({ page }) => { - const oldAnswer = - "This obsolete provisional answer must never be spoken."; - const finalText = "Set the temperature to -10 degrees."; - await assistantOutput(page, oldAnswer, "obsolete-run"); - await nativeEvent(page, { type: "microphoneMute", muted: true }); - await nativeEvent(page, { - type: "user", - ...utterance, - text: finalText, - deliveryAttempts: 0, - }); - await assistantOutput( - page, - " Late obsolete output must also stay unspoken.", - "obsolete-run", - ); - expect(await requests(page)).toHaveLength(1); - expect(await spoken(page)).toEqual(["The reply being interrupted."]); - await completePrompts(page); - await expect.poll(() => requests(page)).toHaveLength(2); - expect((await requests(page))[1].method).toBe("session/prompt"); - await expect(page.getByText(finalText, { exact: true })).toHaveCount(1); - await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( - 0, - ); - const correctedAnswer = - "Only the corrected final answer should be spoken."; - await assistantOutput(page, correctedAnswer, "corrected-run"); - await completePrompts(page); - await expect - .poll(() => spoken(page)) - .toEqual(["The reply being interrupted.", correctedAnswer]); - }, - ); + for (const finalText of [ + "Set the temperature to -10 degrees.", + "set the temperature to 10 degrees.", + ]) + base( + `changed final (${finalText}) never speaks the old answer, including late old output`, + async ({ page }) => { + const oldAnswer = + "This obsolete provisional answer must never be spoken."; + await assistantOutput(page, oldAnswer, "obsolete-run"); + await nativeEvent(page, { type: "microphoneMute", muted: true }); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + await assistantOutput( + page, + " Late obsolete output must also stay unspoken.", + "obsolete-run", + ); + expect(await requests(page)).toHaveLength(1); + expect(await spoken(page)).toEqual(["The reply being interrupted."]); + await completePrompts(page); + await expect.poll(() => requests(page)).toHaveLength(2); + expect((await requests(page))[1].method).toBe("session/prompt"); + await expect(page.getByText(finalText, { exact: true })).toHaveCount(1); + await expect( + page.getByText(utterance.text, { exact: true }), + ).toHaveCount(0); + const correctedAnswer = + "Only the corrected final answer should be spoken."; + await assistantOutput(page, correctedAnswer, "corrected-run"); + await completePrompts(page); + await expect + .poll(() => spoken(page)) + .toEqual(["The reply being interrupted.", correctedAnswer]); + }, + ); base( "recognition ending without a final discards speculative speech and permits the next utterance", From 4101b89f93c9433f0c451d445690cdf7b9244e4c Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 14:12:52 -0400 Subject: [PATCH 09/19] fix(voice): preserve recognition identity through late finals Signed-off-by: John Matthew Tennant --- .../crates/berd-call/src/benchmark/stt.rs | 6 +- src-tauri/crates/berd-call/src/input.rs | 153 ++++++++++++++++-- src-tauri/crates/berd-call/src/main.rs | 8 +- src-tauri/src/commands/native_voice.rs | 71 +++++--- 4 files changed, 206 insertions(+), 32 deletions(-) diff --git a/src-tauri/crates/berd-call/src/benchmark/stt.rs b/src-tauri/crates/berd-call/src/benchmark/stt.rs index 1f4c7de50..0cd6c992c 100644 --- a/src-tauri/crates/berd-call/src/benchmark/stt.rs +++ b/src-tauri/crates/berd-call/src/benchmark/stt.rs @@ -563,7 +563,7 @@ fn observe_event( tracker.speaking_ended = Some(now); } } - Some(VoiceInputEvent::InterimTranscript(_)) => {} + Some(VoiceInputEvent::InterimTranscript { .. }) => {} Some(VoiceInputEvent::RecognitionPendingChanged(active)) => { tracker.pending = active; if active { @@ -576,6 +576,7 @@ fn observe_event( Some(VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. }) => { if tracker.final_text.is_some() { tracker.error = @@ -1044,6 +1045,7 @@ mod tests { let (first_receipt, first_ack) = crate::input::FinalTranscriptStorageReceipt::test_pair(); observe_event( Some(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "stored words".into(), storage_receipt: first_receipt, }), @@ -1058,6 +1060,7 @@ mod tests { crate::input::FinalTranscriptStorageReceipt::test_pair(); assert!(observe_event( Some(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "duplicate words".into(), storage_receipt: duplicate_receipt, }), @@ -1076,6 +1079,7 @@ mod tests { fn late_final_is_a_shutdown_failure_and_is_not_acknowledged() { let (receipt, acknowledgement) = crate::input::FinalTranscriptStorageReceipt::test_pair(); let error = observe_shutdown_event(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "late words".into(), storage_receipt: receipt, }) diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 52fb576b7..771e91da7 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -124,10 +124,13 @@ pub enum VoiceInputEvent { SpeakingChanged(bool), /// Agent speech admission hold, which may expire before recognition finishes. RecognitionPendingChanged(bool), - /// Display only: words Apple is still transcribing, or empty once none are. - /// Never stored, delivered or treated as an utterance. - InterimTranscript(String), + /// Provisional words, identified by their pending recognition rather than its admission hold. + InterimTranscript { + recognition_id: String, + text: String, + }, FinalTranscript { + recognition_id: String, text: String, storage_receipt: FinalTranscriptStorageReceipt, }, @@ -716,6 +719,8 @@ struct PendingRecognitions { ambiguous: bool, /// Where each unresolved turn's speech ended, oldest first; None while it continues. ends: VecDeque>, + /// Stable identities survive admission-hold expiry until recognition resolves. + recognition_ids: VecDeque, /// Where the furthest final so far ended, since one can arrive before its turn ends. #[cfg(any(target_os = "macos", test))] final_end: Option, @@ -726,6 +731,7 @@ struct PendingRecognitions { unfinalized: bool, /// The interim text last shown, so repeats and clears are sent once. interim: String, + interim_recognition_id: String, } impl PendingRecognitions { @@ -747,6 +753,8 @@ impl PendingRecognitions { } else { self.count += 1; self.ends.push_back(None); + self.recognition_ids + .push_back(uuid::Uuid::new_v4().to_string()); } let was_blocking = self.blocking > 0; self.blocking = self.blocking.saturating_add(1).min(limit); @@ -797,12 +805,28 @@ impl PendingRecognitions { /// Interim text is best effort: a full event queue drops it rather than stall recognition. fn show_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { - if self.interim != text { + let recognition_id = self.recognition_ids.back().cloned().unwrap_or_default(); + if self.interim != text + || (!text.is_empty() && self.interim_recognition_id != recognition_id) + { self.interim = text.to_string(); - let _ = events.try_send(VoiceInputEvent::InterimTranscript(self.interim.clone())); + self.interim_recognition_id = recognition_id.clone(); + let _ = events.try_send(VoiceInputEvent::InterimTranscript { + recognition_id, + text: self.interim.clone(), + }); } } + fn take_final_recognition_id(&mut self) -> String { + // Apple can emit several finals before one VAD turn ends. Only the + // first final completes its provisional; later segments need fresh IDs. + let next = uuid::Uuid::new_v4().to_string(); + self.recognition_ids + .front_mut() + .map_or_else(|| next.clone(), |id| std::mem::replace(id, next.clone())) + } + fn result_is_current(&self) -> bool { !self.ambiguous && self.count == self.blocking } @@ -814,6 +838,7 @@ impl PendingRecognitions { let current = self.result_is_current(); self.count -= 1; self.ends.pop_front(); + self.recognition_ids.pop_front(); if current { self.blocking -= 1; if self.blocking == 0 { @@ -887,6 +912,7 @@ fn clamp_vad_frame(samples: &[f32]) -> Vec { fn send_final( events: &tokio_mpsc::Sender, + recognition_id: String, text: String, storage_deadline: Option, ) -> Result<(), ()> { @@ -898,6 +924,7 @@ fn send_final( }; events .blocking_send(VoiceInputEvent::FinalTranscript { + recognition_id, text, storage_receipt: receipt, }) @@ -1488,7 +1515,12 @@ fn complete_parakeet_recognition( storage_deadline: Option, ) { if !text.is_empty() { - let _ = send_final(events, text, storage_deadline); + let _ = send_final( + events, + pending.take_final_recognition_id(), + text, + storage_deadline, + ); } let _ = pending.resolve(events); } @@ -1521,7 +1553,12 @@ fn forward_mac_events( crate::mac_speech::MacSpeechRecognitionEvent::Final(text, end) => { let text = text.trim().to_string(); if !text.is_empty() { - send_final(events, text, storage_deadline)?; + send_final( + events, + pending.take_final_recognition_id(), + text, + storage_deadline, + )?; } // One final can span several turns split by pauses, and one turn // can produce several finals, so a final with an end resolves @@ -1946,7 +1983,12 @@ fn deliver_openai_turns( .then_some(final_storage_deadline) .flatten(); if !text.is_empty() { - let _ = send_final(events, text, storage_deadline); + let _ = send_final( + events, + pending.take_final_recognition_id(), + text, + storage_deadline, + ); } let _ = pending.resolve(events); } @@ -2760,12 +2802,13 @@ mod tests { VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => { assert_eq!(text, "fixture final"); storage_receipt.stored(); observed.push("final"); } - VoiceInputEvent::InterimTranscript(_) => {} + VoiceInputEvent::InterimTranscript { .. } => {} VoiceInputEvent::Failed(error) => panic!("unexpected input failure: {error}"), } } @@ -3103,6 +3146,79 @@ mod tests { ); } + #[test] + fn expired_recognition_keeps_identity_after_new_provisional() { + let (events, mut receiver) = tokio_mpsc::channel(16); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.show_interim(&events, "old provisional"); + pending.release_hold(&events).unwrap(); + pending.begin(&events).unwrap(); + pending.show_interim(&events, "new provisional"); + complete_parakeet_recognition("old final".into(), &events, &mut pending, None); + assert_eq!( + pending.blocking, 1, + "old final must not release the new hold" + ); + complete_parakeet_recognition("new final".into(), &events, &mut pending, None); + let mut provisionals = Vec::new(); + let mut finals = Vec::new(); + while let Ok(event) = receiver.try_recv() { + match event { + VoiceInputEvent::InterimTranscript { + recognition_id, + text, + } => { + provisionals.push((recognition_id, text)); + } + VoiceInputEvent::FinalTranscript { + recognition_id, + text, + .. + } => { + finals.push((recognition_id, text)); + } + _ => {} + } + } + assert_eq!(provisionals.len(), 2); + assert_eq!(finals.len(), 2); + assert_ne!(provisionals[0].0, provisionals[1].0); + assert_eq!(finals[0], (provisionals[0].0.clone(), "old final".into())); + assert_eq!(finals[1], (provisionals[1].0.clone(), "new final".into())); + assert_eq!(pending.blocking, 0); + assert!(pending.recognition_ids.is_empty()); + } + + #[test] + fn identical_interims_on_new_turns_keep_distinct_identities() { + let (events, mut receiver) = tokio_mpsc::channel(8); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.show_interim(&events, "same words"); + pending.release_hold(&events).unwrap(); + pending.begin(&events).unwrap(); + pending.show_interim(&events, "same words"); + let mut ids = Vec::new(); + while let Ok(event) = receiver.try_recv() { + if let VoiceInputEvent::InterimTranscript { recognition_id, .. } = event { + ids.push(recognition_id); + } + } + assert_eq!(ids.len(), 2); + assert_ne!(ids[0], ids[1]); + } + + #[test] + fn several_finals_before_turn_end_do_not_reuse_the_provisional_identity() { + let (events, _receiver) = tokio_mpsc::channel(2); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + let provisional = pending.recognition_ids.front().unwrap().clone(); + assert_eq!(pending.take_final_recognition_id(), provisional); + assert_ne!(pending.take_final_recognition_id(), provisional); + } + #[test] fn final_storage_receipt_unblocks_the_bounded_sender() { let (events, mut receiver) = tokio_mpsc::channel(1); @@ -3110,6 +3226,7 @@ mod tests { let sender = thread::spawn(move || { send_final( &events, + "recognition".into(), "stored words".to_string(), Some(Instant::now() + Duration::from_secs(1)), ) @@ -3119,6 +3236,7 @@ mod tests { let VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } = receiver.blocking_recv().expect("final event") else { panic!("expected final transcript") @@ -3139,6 +3257,7 @@ mod tests { let sender = thread::spawn(move || { send_final( &events, + "recognition".into(), "unstored words".to_string(), Some(Instant::now() + Duration::from_secs(1)), ) @@ -3166,8 +3285,20 @@ mod tests { let (done_tx, done_rx) = mpsc::sync_channel(0); let deadline = Instant::now() + Duration::from_millis(100); let sender = thread::spawn(move || { - send_final(&events, "first".to_string(), Some(deadline)).unwrap(); - send_final(&events, "second".to_string(), Some(deadline)).unwrap(); + send_final( + &events, + "first-id".into(), + "first".to_string(), + Some(deadline), + ) + .unwrap(); + send_final( + &events, + "second-id".into(), + "second".to_string(), + Some(deadline), + ) + .unwrap(); done_tx.send(()).unwrap(); }); let VoiceInputEvent::FinalTranscript { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 4de0b2203..fc2a0ce8a 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -4339,7 +4339,7 @@ fn handle_voice_input_event( )?; update_provisional_suspension(core, active)?; } - VoiceInputEvent::InterimTranscript(text) => write_message( + VoiceInputEvent::InterimTranscript { text, .. } => write_message( writer, &SessionMessage::UserInterim { token: *next_token, @@ -4349,6 +4349,7 @@ fn handle_voice_input_event( VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => store_and_publish_voice_final( text, || storage_receipt.stored(), @@ -8182,7 +8183,10 @@ mod tests { let mut output = Vec::new(); let mut next_token = 7; handle_voice_input_event( - VoiceInputEvent::InterimTranscript("you sh".into()), + VoiceInputEvent::InterimTranscript { + recognition_id: "recognition".into(), + text: "you sh".into(), + }, &mut core, &mut active, &mut next_token, diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index e8db22999..fb0e1a8f0 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1451,6 +1451,7 @@ fn reject_pending_transcript( Ok(TranscriptRejection { attempts, terminal }) } +#[derive(Default)] struct EarlyVoiceTranscript { utterance_id: String, interim_text: String, @@ -1458,17 +1459,6 @@ struct EarlyVoiceTranscript { early_emitted: bool, } -impl Default for EarlyVoiceTranscript { - fn default() -> Self { - Self { - utterance_id: uuid::Uuid::new_v4().to_string(), - interim_text: String::new(), - user_speaking: false, - early_emitted: false, - } - } -} - impl EarlyVoiceTranscript { fn take_ready(&mut self) -> bool { if self.user_speaking || self.early_emitted || self.interim_text.is_empty() { @@ -1478,8 +1468,22 @@ impl EarlyVoiceTranscript { true } + fn set_interim(&mut self, recognition_id: String, text: String) { + if self.utterance_id != recognition_id { + self.reset(); + self.utterance_id = recognition_id; + } + self.interim_text = text.trim().to_string(); + } + + fn finish(&mut self, recognition_id: &str) { + if self.utterance_id == recognition_id { + self.reset(); + } + } + fn reset(&mut self) { - self.utterance_id = uuid::Uuid::new_v4().to_string(); + self.utterance_id.clear(); self.interim_text.clear(); self.early_emitted = false; } @@ -1846,8 +1850,11 @@ pub async fn start_native_voice_conversation( }, ); } - berd_call::input::VoiceInputEvent::InterimTranscript(text) => { - early.interim_text = text.trim().to_string(); + berd_call::input::VoiceInputEvent::InterimTranscript { + recognition_id, + text, + } => { + early.set_interim(recognition_id, text); if early.take_ready() { let _ = event_window.emit( EVENT_NAME, @@ -1862,18 +1869,19 @@ pub async fn start_native_voice_conversation( } } berd_call::input::VoiceInputEvent::FinalTranscript { + recognition_id, text, storage_receipt, } => { let transcript = PendingTranscript { session_id: session_id.clone(), lifecycle_id: lifecycle_id.clone(), - id: early.utterance_id.clone(), + id: recognition_id.clone(), text, revision, delivery_attempts: 0, }; - early.reset(); + early.finish(&recognition_id); let Ok(disposition) = store_final_if_active( &runtime, &pending, @@ -2822,11 +2830,37 @@ mod tests { use super::*; use std::{thread, time::Instant}; + #[test] + fn late_final_keeps_identity_after_hold_expires_and_new_provisional() { + let mut early = EarlyVoiceTranscript::default(); + early.set_interim("old-recognition".into(), "old words".into()); + assert!(early.take_ready()); + let old_provisional = early.utterance_id.clone(); + // RecognitionPendingChanged(false) expires the hold, not recognition. + early.reset(); + early.set_interim("new-recognition".into(), "new words".into()); + assert!(early.take_ready()); + let new_provisional = early.utterance_id.clone(); + assert_ne!(old_provisional, new_provisional); + // The old final arrives while the newer provisional is active. + let old_final = "old-recognition"; + early.finish(old_final); + assert_eq!(old_final, old_provisional); + assert_eq!(early.utterance_id, new_provisional); + assert!( + !early.take_ready(), + "old final must not re-emit the new provisional" + ); + let new_final = "new-recognition"; + early.finish(new_final); + assert_eq!(new_final, new_provisional); + } + #[test] fn recognition_resolution_allows_early_delivery_after_no_final_or_mute() { let mut early = EarlyVoiceTranscript::default(); early.user_speaking = true; - early.interim_text = "first words".into(); + early.set_interim("first-recognition".into(), "first words".into()); assert!(!early.take_ready()); early.user_speaking = false; assert!(early.take_ready()); @@ -2835,7 +2869,7 @@ mod tests { early.reset(); assert_ne!(early.utterance_id, previous_id); assert!(!early.take_ready()); - early.interim_text = "next words".into(); + early.set_interim("next-recognition".into(), "next words".into()); assert!(early.take_ready()); } @@ -3596,6 +3630,7 @@ mod tests { VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => { storage_receipt.stored(); if text == "marker" { From d893820ac7b09b8ea89f598780f6467bfb9ffb12 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 14:12:52 -0400 Subject: [PATCH 10/19] fix(voice): isolate and bound provisional preparation Signed-off-by: John Matthew Tennant --- .../__tests__/acpNotificationHandler.test.ts | 33 -- src/features/chat/hooks/useChat.ts | 1 - src/features/chat/lib/backgroundSend.test.ts | 28 -- src/features/chat/lib/backgroundSend.ts | 1 - src/features/chat/lib/sendCore.ts | 4 - src/features/chat/lib/steerCore.ts | 1 - src/features/chat/types.ts | 2 - .../useVoiceConversationController.test.ts | 254 ------------ .../hooks/useVoiceConversationController.ts | 204 +++------- .../lib/nativeAssistantSpeech.test.ts | 183 --------- .../lib/nativeAssistantSpeech.ts | 23 +- .../lib/privateVoiceDraft.test.ts | 107 +++++ .../lib/privateVoiceDraft.ts | 92 +++++ .../stores/voiceConversationStore.test.ts | 47 --- .../stores/voiceConversationStore.ts | 26 +- src/shared/api/__tests__/acp.test.ts | 40 -- src/shared/api/__tests__/acpBackends.test.ts | 25 ++ src/shared/api/acp.ts | 23 +- src/shared/api/acpApi.ts | 12 + src/shared/api/acpConnection.ts | 9 + tests/e2e/fixtures/tauri-mock.ts | 36 +- tests/e2e/voice-conversation.spec.ts | 384 ++++++++++-------- 22 files changed, 537 insertions(+), 998 deletions(-) create mode 100644 src/features/voice-conversation/lib/privateVoiceDraft.test.ts create mode 100644 src/features/voice-conversation/lib/privateVoiceDraft.ts diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index 38f8e0b75..b5de9856a 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -2183,39 +2183,6 @@ describe("acpNotificationHandler", () => { }); }); - it.each([ - "provisional words", - "corrected words", - ])("replay exposes only the persistent final voice turn (%s)", async (finalText) => { - const sessionId = `early-voice-replay-${finalText}`; - useChatStore.setState({ loadingSessionIds: new Set([sessionId]) }); - for (const [messageId, text, hidden] of [ - ["provisional", "Provisional speech recognition.", true], - ["provisional", "provisional words", true], - ["final", "Final transcript, do not repeat an unchanged answer.", true], - ["final", finalText, false], - ] as const) { - await handleSessionNotification({ - sessionId, - update: { - sessionUpdate: "user_message_chunk", - messageId, - content: { - type: "text", - text, - ...(hidden ? { annotations: { audience: ["assistant"] } } : {}), - }, - }, - } as never); - } - expect(getReplayBuffer(sessionId)).toHaveLength(1); - expect(getReplayBuffer(sessionId)?.[0]).toMatchObject({ - id: "final", - role: "user", - content: [{ type: "text", text: finalText }], - }); - }); - it("replay restores skill chips from assistant-only user chunks", async () => { const replaySessionId = "replay-skill-session"; useChatStore.setState({ diff --git a/src/features/chat/hooks/useChat.ts b/src/features/chat/hooks/useChat.ts index e758cb764..59da52eeb 100644 --- a/src/features/chat/hooks/useChat.ts +++ b/src/features/chat/hooks/useChat.ts @@ -206,7 +206,6 @@ export function useChat( persona: effectivePersonaInfo, attachments, assistantPrompt: sendOptions?.assistantPrompt, - assistantOnlyInput: sendOptions?.assistantOnlyInput, displayText: sendOptions?.displayText, userMessageId: sendOptions?.userMessageId, chips: sendOptions?.chips, diff --git a/src/features/chat/lib/backgroundSend.test.ts b/src/features/chat/lib/backgroundSend.test.ts index 4edd40511..6a4073a0d 100644 --- a/src/features/chat/lib/backgroundSend.test.ts +++ b/src/features/chat/lib/backgroundSend.test.ts @@ -43,34 +43,6 @@ describe("sendPromptInBackground", () => { ); }); - it("preserves assistant-only input through queued background dispatch", async () => { - await sendPromptInBackground( - "session-1", - "provisional words", - "goose", - undefined, - { - assistantOnlyInput: true, - userMessageMetadata: { - origin: "voice_conversation", - userVisible: false, - }, - }, - ); - - expect(mocks.dispatchPrompt).toHaveBeenCalledWith( - "session-1", - "provisional words", - expect.objectContaining({ - assistantOnlyInput: true, - userMessageMetadata: { - origin: "voice_conversation", - userVisible: false, - }, - }), - ); - }); - it.each([ new QueuedSessionNotReadyError(), new QueuedMessageOwnershipLostError(), diff --git a/src/features/chat/lib/backgroundSend.ts b/src/features/chat/lib/backgroundSend.ts index 71ba31691..fcd8e3077 100644 --- a/src/features/chat/lib/backgroundSend.ts +++ b/src/features/chat/lib/backgroundSend.ts @@ -44,7 +44,6 @@ export function sendPromptInBackground( : undefined, attachments, assistantPrompt: sendOptions.assistantPrompt, - assistantOnlyInput: sendOptions.assistantOnlyInput, displayText: sendOptions.displayText, chips: sendOptions.chips, userMessageMetadata: sendOptions.userMessageMetadata, diff --git a/src/features/chat/lib/sendCore.ts b/src/features/chat/lib/sendCore.ts index fa834210a..b2f5e4dde 100644 --- a/src/features/chat/lib/sendCore.ts +++ b/src/features/chat/lib/sendCore.ts @@ -55,8 +55,6 @@ export interface SendCoreOptions { attachments?: ChatAttachmentDraft[]; /** Assistant-audience prompt (skills/builder) merged ahead of the user text. */ assistantPrompt?: string; - /** Internal provisional context, persisted for the assistant only. */ - assistantOnlyInput?: boolean; /** Text shown in the transcript when it differs from the prompt sent. */ displayText?: string; /** Reuses a provisional local transcript row when the send commits. */ @@ -233,7 +231,6 @@ export async function dispatchPrompt( const tSendStart = performance.now(); const { assistantPrompt, - assistantOnlyInput, acpGooseMetadata, attachments, background, @@ -420,7 +417,6 @@ export async function dispatchPrompt( const promptPromise = acpSendMessage(sessionId, acpPrompt, { systemPrompt, ...(assistantPrompt ? { assistantPrompt } : {}), - assistantOnlyInput, personaId: persona?.id, personaName: persona?.name, goose: acpGooseMetadata, diff --git a/src/features/chat/lib/steerCore.ts b/src/features/chat/lib/steerCore.ts index 3f182ba29..c9a63ff90 100644 --- a/src/features/chat/lib/steerCore.ts +++ b/src/features/chat/lib/steerCore.ts @@ -137,7 +137,6 @@ export async function steerPromptInSession( ...(sendOptions?.assistantPrompt ? { assistantPrompt: sendOptions.assistantPrompt } : {}), - assistantOnlyInput: sendOptions?.assistantOnlyInput, goose: sendOptions?.acpGooseMetadata, images: images?.map( (img) => [img.base64, img.mimeType] as [string, string], diff --git a/src/features/chat/types.ts b/src/features/chat/types.ts index c411da8bc..de5b21cda 100644 --- a/src/features/chat/types.ts +++ b/src/features/chat/types.ts @@ -73,8 +73,6 @@ export interface ChatSendOptions { /** Reuses a provisional local transcript row when the send commits. */ userMessageId?: string; assistantPrompt?: string; - /** Internal provisional context, persisted for the assistant only. */ - assistantOnlyInput?: boolean; chips?: MessageChip[]; userMessageMetadata?: Partial; acpGooseMetadata?: Record; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 517544e14..6752807b8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -92,260 +92,6 @@ function deferred() { } describe("voice transcript delivery coordination", () => { - it.each([ - "provisional words", - "provisional words?", - "provisional words after mute", - "corrected words", - "corrected words after mute", - "set the temperature to -10 degrees", - "calculate 3 + 4", - "open foo.ts", - ])("reconciles early speech once (%s)", async (finalText) => { - const changed = ![ - "provisional words", - "provisional words?", - "provisional words after mute", - ].includes(finalText); - const onSend = vi.fn(async (text, _persona, _attachments, options) => { - const state = useChatStore.getState(); - const existing = state.messagesBySession["session-1"]?.find( - (message) => message.id === options.userMessageId, - ); - const message = { - id: - existing?.id ?? - `row-${state.messagesBySession["session-1"]?.length ?? 0}`, - role: "user" as const, - created: 1, - content: [{ type: "text" as const, text }], - metadata: options.userMessageMetadata, - }; - if (existing) - state.updateMessage("session-1", existing.id, () => message); - else state.addMessage("session-1", message); - return true; - }); - useVoiceConversationStore.setState({ - status: { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - microphoneMuted: false, - revision: 1, - }, - earlyTranscriptPending: null, - hydrated: true, - init: vi.fn().mockResolvedValue(undefined), - drainPendingTranscripts: vi.fn().mockResolvedValue(undefined), - }); - const { unmount } = renderHook(() => - useVoiceConversationController({ - sessionId: "session-1", - onSend, - enabled: true, - isGooseSession: true, - pocketReady: true, - onPocketSetupRequired: vi.fn(), - }), - ); - const event = { - sessionId: "session-1", - lifecycleId: "lifecycle-1", - id: `early-test-${finalText}`, - revision: 1, - text: finalText.includes("temperature") - ? "set the temperature to 10 degrees" - : finalText.includes("calculate") - ? "calculate 3 - 4" - : finalText === "open foo.ts" - ? "open Foo.ts" - : "provisional words", - }; - await act(async () => { - await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); - }); - expect(onSend).toHaveBeenCalledTimes(1); - expect( - useVoiceConversationStore.getState().earlyTranscriptPending, - ).not.toBeNull(); - expect( - useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata - ?.userVisible, - ).toBe(false); - const final = { - ...event, - type: "user", - text: - finalText === "provisional words after mute" - ? "provisional words" - : finalText, - deliveryAttempts: 0, - }; - if (finalText.endsWith("after mute")) { - await act(async () => { - useVoiceConversationStore.setState({ microphoneMuted: true }); - }); - expect( - useVoiceConversationStore.getState().earlyTranscriptPending, - ).not.toBeNull(); - } - if (finalText.startsWith("corrected words")) { - useChatStore.getState().setActiveRunId("session-1", "speculative-run"); - useChatStore.getState().setChatState("session-1", "streaming"); - let finalDelivery: void | Promise; - await act(async () => { - finalDelivery = voiceStoreMocks.subscriber?.(final); - await Promise.resolve(); - }); - expect(onSend).toHaveBeenCalledTimes(1); - // Old output arriving while the final is waiting retains provisional causality. - expect( - useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata - ?.voiceUtteranceId, - ).toBe(`${event.id}:early`); - await act(async () => { - useChatStore.getState().setChatState("session-1", "idle"); - useChatStore.getState().settleActiveRun("session-1"); - await finalDelivery; - }); - } else { - await act(async () => { - await voiceStoreMocks.subscriber?.(final); - }); - } - await act(async () => { - await voiceStoreMocks.subscriber?.(final); - }); - expect(onSend).toHaveBeenCalledTimes(2); - expect(onSend.mock.calls[0]?.[3]).toMatchObject({ - assistantOnlyInput: true, - }); - expect(onSend.mock.calls[1]?.[3]).toMatchObject({ - assistantOnlyInput: false, - }); - const rows = useChatStore.getState().messagesBySession["session-1"] ?? []; - expect( - rows.filter((row) => row.metadata?.userVisible !== false), - ).toHaveLength(1); - expect(rows[1]).toMatchObject({ - content: [{ type: "text", text: final.text }], - metadata: { userVisible: true, voiceUtteranceId: event.id }, - }); - expect( - useVoiceConversationStore.getState().earlyTranscriptPending, - ).toBeNull(); - expect( - useVoiceConversationStore.getState().earlyTranscriptResolution?.accepted, - ).toBe(!changed); - act(() => useVoiceConversationStore.setState({ microphoneMuted: false })); - unmount(); - }); - - it.each([ - false, - true, - ])("releases no-final provisional speech even when muted (%s)", async (muted) => { - const onSend = vi.fn(async (text, _persona, _attachments, options) => { - const state = useChatStore.getState(); - const existing = state.messagesBySession["session-1"]?.find( - (message) => message.id === options.userMessageId, - ); - const message = { - id: - existing?.id ?? - `row-${state.messagesBySession["session-1"]?.length ?? 0}`, - role: "user" as const, - created: 1, - content: [{ type: "text" as const, text }], - metadata: options.userMessageMetadata, - }; - if (existing) - state.updateMessage("session-1", existing.id, () => message); - else state.addMessage("session-1", message); - return true; - }); - useVoiceConversationStore.setState({ - status: { - available: true, - unavailableReason: null, - lifecycle: "running", - sessionId: "session-1", - ownerWindowLabel: "main", - microphoneMuted: false, - revision: 1, - }, - earlyTranscriptPending: null, - hydrated: true, - init: vi.fn().mockResolvedValue(undefined), - drainPendingTranscripts: vi.fn().mockResolvedValue(undefined), - }); - const { unmount } = renderHook(() => - useVoiceConversationController({ - sessionId: "session-1", - onSend, - enabled: true, - isGooseSession: true, - pocketReady: true, - onPocketSetupRequired: vi.fn(), - }), - ); - const event = { - sessionId: "session-1", - lifecycleId: "lifecycle-1", - id: "abandoned-test", - revision: 1, - text: "provisional words", - }; - await act(async () => { - await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); - }); - expect(onSend).toHaveBeenCalledTimes(1); - expect( - useVoiceConversationStore.getState().earlyTranscriptPending, - ).not.toBeNull(); - expect( - useChatStore.getState().messagesBySession["session-1"]?.[0]?.metadata - ?.userVisible, - ).toBe(false); - - await act(async () => { - if (muted) useVoiceConversationStore.setState({ microphoneMuted: true }); - await voiceStoreMocks.subscriber?.({ - type: "recognitionPending", - sessionId: event.sessionId, - revision: event.revision, - pending: false, - }); - }); - expect( - useVoiceConversationStore.getState().earlyTranscriptPending, - ).toBeNull(); - expect( - useVoiceConversationStore.getState().earlyTranscriptResolution?.accepted, - ).toBe(false); - await act(async () => { - useVoiceConversationStore.setState({ microphoneMuted: false }); - await voiceStoreMocks.subscriber?.({ - ...event, - id: "next-utterance", - type: "interimUser", - }); - }); - expect(onSend).toHaveBeenCalledTimes(2); - await act(async () => { - await voiceStoreMocks.subscriber?.({ - type: "recognitionPending", - sessionId: event.sessionId, - revision: event.revision, - pending: false, - }); - }); - unmount(); - }); - it("suppresses floating controls only for the focused owner session", () => { const base = { activeSessionId: "session-1", diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index ace373834..49434cd01 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -6,6 +6,11 @@ import type { ChatInputSendHandler, ChatInputVoiceConversation, } from "@/features/chat/types"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { + startPrivateVoiceDraft, + type PrivateVoiceDraft, +} from "../lib/privateVoiceDraft"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; import { createSystemNotificationMessage } from "@/shared/types/messages"; @@ -370,42 +375,6 @@ function voiceDeliveryOpportunity( return null; } -export function waitForVoiceRunSettlement(sessionId: string): Promise { - const settled = () => { - const runtime = useChatStore.getState().getSessionRuntime(sessionId); - return ( - runtime.activeRunId === null && - runtime.chatState === "idle" && - !runtime.isRunCancellationPending - ); - }; - if (settled()) return Promise.resolve(); - return new Promise((resolve, reject) => { - const cleanup = () => { - unsubscribeChat(); - unsubscribeVoice(); - }; - const unsubscribeChat = useChatStore.subscribe(() => { - if (!settled()) return; - cleanup(); - resolve(); - }); - const unsubscribeVoice = useVoiceConversationStore.subscribe((voice) => { - if ( - voice.status.lifecycle === "running" && - voice.status.sessionId === sessionId - ) - return; - cleanup(); - reject( - new VoiceTranscriptDeferredError( - "Voice transcript is waiting for its speculative run to settle.", - ), - ); - }); - }); -} - export function waitForVoiceDeliveryOpportunity( sessionId: string, ): Promise { @@ -560,23 +529,9 @@ export function resetVoiceUiWhenRunSettles( queueMicrotask(check); } -interface EarlyDelivery { - text: string; - key: string; - admitted: boolean; -} -const earlyDeliveries = new Map(); +const earlyDeliveries = new Map(); const finalTranscriptsSeen = new Set(); -function sameVoiceWords(first: string, second: string): boolean { - const words = (text: string) => - text - .trim() - .replace(/[.!?]+$/u, "") - .replace(/\s+/gu, " "); - return words(first) === words(second); -} - function voiceEventKey(event: { sessionId: string; lifecycleId: string; @@ -605,20 +560,15 @@ function ensureVoiceEventDeliveryInitialized() { deliveryInitialized = true; observeChainedVoiceStatus(); useVoiceConversationStore.subscribe((state, previous) => { - if (state.earlyTranscriptResolution?.terminal === true) { - earlyDeliveries.delete(state.earlyTranscriptResolution.finalKey); - finalTranscriptsSeen.delete(state.earlyTranscriptResolution.finalKey); - } if ( state.status.revision === previous.status.revision && state.status.sessionId === previous.status.sessionId && state.status.lifecycle === previous.status.lifecycle ) return; + for (const draft of earlyDeliveries.values()) draft.dispose(); earlyDeliveries.clear(); finalTranscriptsSeen.clear(); - if (state.earlyTranscriptPending !== null) - useVoiceConversationStore.setState({ earlyTranscriptPending: null }); }); subscribeToVoiceConversationEvents(async (incoming) => { const isEarly = incoming.type === "interimUser"; @@ -642,19 +592,11 @@ function ensureVoiceEventDeliveryInitialized() { if (event.type === "recognitionPending") { if (event.pending) return; if (!earlyEventIsCurrent(event, false)) return; - const pending = - useVoiceConversationStore.getState().earlyTranscriptPending; - // A final already queued owns reconciliation. Otherwise recognition has - // ended without a final. Release the gate even if admission is waiting. - if (!pending || finalTranscriptsSeen.has(pending)) return; - const early = earlyDeliveries.get(pending); - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: early - ? { key: early.key, finalKey: pending, accepted: false } - : null, - }); - earlyDeliveries.delete(pending); + for (const [key, draft] of earlyDeliveries) { + if (finalTranscriptsSeen.has(key)) continue; + draft.dispose(); + earlyDeliveries.delete(key); + } return; } if (event.type === "activity") return; @@ -671,37 +613,35 @@ function ensureVoiceEventDeliveryInitialized() { } const eventKey = voiceEventKey(event); - if (!isEarly) { - // Bounded by lifecycle, and retained only while an early delivery exists. - if (earlyDeliveries.has(eventKey)) finalTranscriptsSeen.add(eventKey); - } const deliveryRevision = event.revision; const shouldNotifyFailure = event.deliveryAttempts === 0; if (isEarly) { if ( !earlyEventIsCurrent(event) || earlyDeliveries.has(eventKey) || - useVoiceConversationStore.getState().earlyTranscriptPending !== null + activeRouteIsBlocked(event.sessionId) || + activeSendRoute?.sessionId !== event.sessionId ) return; - earlyDeliveries.set(eventKey, { - text: event.text, - key: voiceEventKey({ ...event, id: `${event.id}:early` }), - admitted: false, - }); - useVoiceConversationStore.setState({ earlyTranscriptPending: eventKey }); + const session = useChatSessionStore + .getState() + .sessions.find((session) => session.id === event.sessionId); + if (!session?.workingDir) return; + for (const [key, draft] of earlyDeliveries) { + if (finalTranscriptsSeen.has(key)) continue; + draft.dispose(); + earlyDeliveries.delete(key); + } + // Independent of the authoritative queue, even when the main run is busy. + earlyDeliveries.set( + eventKey, + startPrivateVoiceDraft(event.sessionId, session.workingDir, event.text), + ); + return; } + if (earlyDeliveries.has(eventKey)) finalTranscriptsSeen.add(eventKey); return enqueueVoiceTranscriptDelivery(event.sessionId, async () => { const early = earlyDeliveries.get(eventKey); - if ( - isEarly && - (!early || - !earlyEventIsCurrent(event) || - finalTranscriptsSeen.has(eventKey)) - ) { - return; - } - if (isEarly && activeRouteIsBlocked(event.sessionId)) return; if (activeRouteIsBlocked(event.sessionId)) { throw new VoiceTranscriptDeferredError( "Voice transcript is waiting for its bound chat to become available.", @@ -710,7 +650,6 @@ function ensureVoiceEventDeliveryInitialized() { const route = activeSendRoute; if (!route || route.sessionId !== event.sessionId) { - if (isEarly) return; const message = "Voice transcript could not be sent because its bound chat is unavailable."; if (shouldNotifyFailure) addErrorNotification(event.sessionId, message); @@ -718,53 +657,29 @@ function ensureVoiceEventDeliveryInitialized() { } const store = useVoiceConversationStore.getState(); - if (!isEarly) store.setUiState("user-speaking"); - const releaseEarly = (accepted: boolean) => { - if (!early) return; - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: { - key: early.key, - finalKey: eventKey, - accepted, - }, - }); - earlyDeliveries.delete(eventKey); - finalTranscriptsSeen.delete(eventKey); - }; + store.setUiState("user-speaking"); + const draft = await early?.take(); + earlyDeliveries.delete(eventKey); + finalTranscriptsSeen.delete(eventKey); const sendOptions = { - assistantOnlyInput: isEarly, userMessageMetadata: { - ...(isEarly ? { userVisible: false } : { userVisible: true }), + userVisible: true, origin: "voice_conversation" as const, - voiceUtteranceId: isEarly ? `${event.id}:early` : event.id, + voiceUtteranceId: event.id, voiceConversationLifecycleId: event.lifecycleId, voiceConversationRevision: event.revision, }, acpGooseMetadata: { origin: "voice_conversation", - voiceUtteranceId: isEarly ? `${event.id}:early` : event.id, + voiceUtteranceId: event.id, voiceConversationLifecycleId: event.lifecycleId, voiceConversationRevision: event.revision, }, }; try { - // Keep the provisional row and its causal identity immutable. A changed - // final must not become the cause of late messages from the old run. - if ( - !isEarly && - early?.admitted && - !sameVoiceWords(early.text, event.text) - ) - await waitForVoiceRunSettlement(event.sessionId); - // This runs inside the per-session queue, so a prior send can change - // the opportunity to steer before the next transcript is evaluated. - const opportunity = isEarly - ? voiceDeliveryOpportunity(event.sessionId) - : await waitForVoiceDeliveryOpportunity(event.sessionId); - // Provisional delivery is best effort. Never block retained finals or - // the next utterance waiting for a run that has not started yet. - if (!opportunity) return; + const opportunity = await waitForVoiceDeliveryOpportunity( + event.sessionId, + ); if (activeRouteIsBlocked(event.sessionId)) { throw new VoiceTranscriptDeferredError( "Voice transcript is waiting for its bound chat to become available.", @@ -776,31 +691,20 @@ function ensureVoiceEventDeliveryInitialized() { "Voice transcript could not be sent because its bound chat is unavailable.", ); } - if ( - isEarly && - (!early || - !earlyEventIsCurrent(event) || - finalTranscriptsSeen.has(eventKey)) - ) - return; const playbackNotice = takeVoicePlaybackNotices(event.sessionId); const displayOptions = { ...sendOptions, assistantPrompt: [ playbackNotice, - isEarly - ? "The user paused. This is provisional speech recognition and may be corrected. Do not treat it as confirmation for irreversible actions." - : early?.admitted - ? sameVoiceWords(early.text, event.text) - ? "This is the final transcript of the provisional speech already sent. It is unchanged. Persist this user turn, but do not repeat an answer already given or in progress unless the request changed." - : "Speech recognition correction: the final user transcript replaces the provisional words in the prior voice turn. Discard any answer based on the provisional wording and answer the corrected request." - : null, + draft + ? `Private provisional DRAFT (not spoken, not executed, may be wrong):\n${draft}\n\nThe user transcript in this turn is authoritative. Answer it and perform any requested actions normally. Use the draft only if it is relevant to the final transcript.` + : null, ] .filter(Boolean) .join("\n\n"), displayText: event.text, }; - if (!isEarly) store.setUiState("agent-working"); + store.setUiState("agent-working"); const delivered = opportunity === "steer" ? await steerPromptInSession( @@ -819,9 +723,9 @@ function ensureVoiceEventDeliveryInitialized() { if (delivered === false) { // Runtime state may have changed between the opportunity read and // send admission. Re-evaluate once, then steer if a run now exists. - const retryOpportunity = isEarly - ? voiceDeliveryOpportunity(event.sessionId) - : await waitForVoiceDeliveryOpportunity(event.sessionId); + const retryOpportunity = await waitForVoiceDeliveryOpportunity( + event.sessionId, + ); if (retryOpportunity !== "steer") { throw new Error( "Voice transcript was not accepted by the chat session.", @@ -835,24 +739,10 @@ function ensureVoiceEventDeliveryInitialized() { { throwOnError: true }, ); } - if (isEarly && early) early.admitted = true; - if (!isEarly) - releaseEarly( - Boolean(early?.admitted && sameVoiceWords(early.text, event.text)), - ); // Admission releases the transcript queue so later speech can steer. // UI working state follows the actual chat runtime independently. resetVoiceUiWhenRunSettles(event.sessionId, deliveryRevision); } catch (deliveryError) { - if (isEarly) { - // Early delivery is best effort. Retained final delivery remains the - // authoritative recovery path. Keep the speech gate until then. - console.warn( - "Provisional voice delivery was not admitted", - deliveryError, - ); - return; - } const current = useVoiceConversationStore.getState(); if (deliveryError instanceof VoiceTranscriptDeferredError) { if ( diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index d044b37c5..6385aee21 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -271,8 +271,6 @@ describe("native assistant speech stream", () => { assistantSpeaking: false, microphoneMuted: false, latestFinalizedTranscriptKey: null, - earlyTranscriptPending: null, - earlyTranscriptResolution: null, }); }); @@ -507,187 +505,6 @@ describe("native assistant speech stream", () => { ).toMatchObject({ text: source }); }); - it.each([ - false, - true, - ])("holds speculative output and resolves its final (accepted: %s)", async (accepted) => { - const earlyKey = ["session-1", "lifecycle-1", "1", "early:early"].join( - "\0", - ); - const finalKey = ["session-1", "lifecycle-1", "1", "early"].join("\0"); - startNativeAssistantSpeech("session-1", vi.fn()); - useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); - useChatStore - .getState() - .setMessages("session-1", [ - voiceUser("early:early", { userVisible: false }), - assistant( - [{ type: "text", text: "Speculative answer." }], - "completed", - "early-answer", - ), - ]); - await new Promise((resolve) => window.setTimeout(resolve, 15)); - expect(mocks.start).not.toHaveBeenCalled(); - useVoiceConversationStore.setState({ microphoneMuted: true }); - finalizeVoiceTranscript("early"); - await new Promise((resolve) => window.setTimeout(resolve, 15)); - expect(mocks.start).not.toHaveBeenCalled(); - // The provisional row is immutable. A late message from that same run - // arrives before changed-final dispatch is allowed to begin. - useChatStore - .getState() - .addMessage( - "session-1", - assistant( - [{ type: "text", text: "Late speculative answer." }], - "completed", - "late-early-answer", - ), - ); - useChatStore.getState().addMessage("session-1", voiceUser("early")); - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: { key: earlyKey, finalKey, accepted }, - }); - if (accepted) { - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - expect.any(String), - "Speculative answer.", - ), - ); - } else { - await new Promise((resolve) => window.setTimeout(resolve, 15)); - expect(mocks.start).not.toHaveBeenCalled(); - useChatStore - .getState() - .addMessage( - "session-1", - assistant( - [{ type: "text", text: "Corrected answer." }], - "completed", - "corrected-answer", - ), - ); - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - expect.any(String), - "Corrected answer.", - ), - ); - expect( - mocks.append.mock.calls.every( - ([, text]) => !text.toLowerCase().includes("speculative"), - ), - ).toBe(true); - } - }); - - it("releases an early reply after a real speaking-to-pause-to-final sequence", async () => { - const earlyKey = ["session-1", "lifecycle-1", "1", "paused:early"].join( - "\0", - ); - const finalKey = ["session-1", "lifecycle-1", "1", "paused"].join("\0"); - startNativeAssistantSpeech("session-1", vi.fn()); - useVoiceConversationStore.setState({ userSpeaking: true }); - useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); - useChatStore - .getState() - .setMessages("session-1", [ - voiceUser("paused:early", { userVisible: false }), - assistant( - [{ type: "text", text: "Early paused answer." }], - "completed", - "paused-answer", - ), - ]); - useVoiceConversationStore.setState({ userSpeaking: false }); - await new Promise((resolve) => window.setTimeout(resolve, 300)); - expect(mocks.start).not.toHaveBeenCalled(); - finalizeVoiceTranscript("paused"); - useChatStore.getState().addMessage("session-1", voiceUser("paused")); - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: { key: earlyKey, finalKey, accepted: true }, - }); - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - expect.any(String), - "Early paused answer.", - ), - ); - }); - - it("resolves interrupted playback recognition while early speech is gated", async () => { - startNativeAssistantSpeech("session-1", vi.fn()); - useChatStore - .getState() - .setMessages("session-1", [ - assistant([{ type: "text", text: "Interrupted reply." }]), - ]); - await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); - emit("started"); - useVoiceConversationStore.setState({ userSpeaking: true }); - await vi.waitFor(() => expect(mocks.stop).toHaveBeenCalled()); - emit("interrupted"); - - const finalKey = ["session-1", "lifecycle-1", "1", "interrupt"].join("\0"); - const earlyKey = `${finalKey}:early`; - useVoiceConversationStore.setState({ earlyTranscriptPending: finalKey }); - useVoiceConversationStore.setState({ userSpeaking: false }); - useChatStore - .getState() - .addMessage( - "session-1", - voiceUser("interrupt:early", { userVisible: false }), - ); - useChatStore - .getState() - .addMessage( - "session-1", - assistant( - [{ type: "text", text: "Reply to the interruption." }], - "completed", - "interruption-answer", - ), - ); - finalizeVoiceTranscript("interrupt"); - useChatStore.getState().addMessage("session-1", voiceUser("interrupt")); - expect(mocks.start).toHaveBeenCalledTimes(1); - useVoiceConversationStore.setState({ - earlyTranscriptPending: null, - earlyTranscriptResolution: { key: earlyKey, finalKey, accepted: true }, - }); - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - expect.any(String), - "Reply to the interruption.", - ), - ); - mocks.streamHandler?.({ - streamId: mocks.start.mock.calls[1]?.[0] as string, - state: "completed", - error: null, - }); - useChatStore - .getState() - .addMessage( - "session-1", - assistant( - [{ type: "text", text: "Later replies still speak." }], - "completed", - "later-answer", - ), - ); - await vi.waitFor(() => - expect(mocks.append).toHaveBeenCalledWith( - expect.any(String), - "Later replies still speak.", - ), - ); - }); - it("pushes raw assistant deltas without frontend sentence segmentation", async () => { startNativeAssistantSpeech("session-1", vi.fn()); useChatStore diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 258978ad7..7ea41ce9d 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1433,17 +1433,6 @@ export function startNativeAssistantSpeech( } const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; cacheCausalTranscriptKeys(messages); - const resolution = voice.earlyTranscriptResolution; - if (resolution?.accepted) { - for (const [id, key] of causalTranscriptKeyByMessage) { - if (key === resolution.key) - causalTranscriptKeyByMessage.set(id, resolution.finalKey); - } - } - // Do not consume or invalidate speculative text while recognition or its - // final delivery is unresolved. The controller releases this gate only - // after the final row and the agent's correction have been admitted. - if (voice.earlyTranscriptPending !== null) return; if ( heldSpeech || voice.userSpeaking || @@ -1804,11 +1793,7 @@ export function startNativeAssistantSpeech( inspect(); return; } - if ( - finalizedTranscriptChanged && - !pendingUserRecognitionSegment && - voice.earlyTranscriptPending === null - ) { + if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) { if (heldReleaseTimer !== null) { window.clearTimeout(heldReleaseTimer); heldReleaseTimer = null; @@ -1837,12 +1822,6 @@ export function startNativeAssistantSpeech( return; } if (finalizedTranscriptChanged && pendingUserRecognitionSegment) { - if (voice.earlyTranscriptPending !== null) { - resolvePendingRecognitionSegment(false); - discardResumableInterruption(); - inspect(); - return; - } holdAssistantChanges( useChatStore.getState().messagesBySession[sessionId] ?? [], ); diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts new file mode 100644 index 000000000..982682655 --- /dev/null +++ b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { startPrivateVoiceDraft } from "./privateVoiceDraft"; +const mocks = vi.hoisted(() => ({ + forkSession: vi.fn(), + setSessionMode: vi.fn(), + prompt: vi.fn(), + cancelSession: vi.fn(), + deleteSession: vi.fn(), + hide: vi.fn(), + intercept: vi.fn(), +})); +vi.mock("@/shared/api/acpApi", () => mocks); +vi.mock("@/shared/api/acpConnection", () => ({ + hidePrivateSessionNotifications: mocks.hide, + interceptSessionNotifications: mocks.intercept, +})); +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} +beforeEach(() => { + vi.useFakeTimers(); + vi.resetAllMocks(); + mocks.forkSession.mockResolvedValue({ sessionId: "private" }); + mocks.setSessionMode.mockResolvedValue(undefined); + mocks.prompt.mockResolvedValue(undefined); + mocks.cancelSession.mockResolvedValue(undefined); + mocks.deleteSession.mockResolvedValue(undefined); + mocks.intercept.mockReturnValue(vi.fn()); +}); +afterEach(() => vi.useRealTimers()); +it("awaits chat mode before first prompt and captures only private text", async () => { + const mode = deferred(); + mocks.setSessionMode.mockReturnValue(mode.promise); + mocks.prompt.mockImplementation(async () => { + mocks.intercept.mock.calls[0][0]({ + sessionId: "private", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Draft" }, + }, + }); + }); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.setSessionMode).toHaveBeenCalledWith("private", "chat"); + expect(mocks.prompt).not.toHaveBeenCalled(); + mode.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(await draft.take()).toBe("Draft"); + expect(mocks.forkSession).toHaveBeenCalledWith("main", "/work"); + expect(mocks.cancelSession).toHaveBeenCalledWith("private"); +}); +it("fails closed when chat mode is rejected", async () => { + mocks.setSessionMode.mockRejectedValue(new Error("unsupported")); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await expect(draft.take()).resolves.toBeNull(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it.each([ + "fork", + "mode", + "prompt", +])("bounds stalled %s and cancellation without touching main", async (stage) => { + const stalled = deferred(); + mocks[ + stage === "fork" + ? "forkSession" + : stage === "mode" + ? "setSessionMode" + : "prompt" + ].mockReturnValue(stalled.promise); + mocks.cancelSession.mockReturnValue(stalled.promise); + mocks.deleteSession.mockReturnValue(stalled.promise); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + const taken = draft.take(); + await vi.advanceTimersByTimeAsync(750); + expect(await taken).toBeNull(); + expect(mocks.cancelSession.mock.calls.every(([id]) => id === "private")).toBe( + true, + ); + if (stage !== "prompt") expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("bounds best effort setup and cleans a fork created after disposal", async () => { + const fork = deferred<{ sessionId: string }>(); + mocks.forkSession.mockReturnValue(fork.promise); + startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(2000); + fork.resolve({ sessionId: "late-private" }); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.cancelSession).toHaveBeenCalledWith("late-private"); + expect(mocks.hide).toHaveBeenCalledWith("late-private"); + expect(mocks.setSessionMode).not.toHaveBeenCalled(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("never prompts after a timed out mode ack", async () => { + const mode = deferred(); + mocks.setSessionMode.mockReturnValue(mode.promise); + startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(2000); + mode.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.ts b/src/features/voice-conversation/lib/privateVoiceDraft.ts new file mode 100644 index 000000000..c11a48d4a --- /dev/null +++ b/src/features/voice-conversation/lib/privateVoiceDraft.ts @@ -0,0 +1,92 @@ +import { + cancelSession, + deleteSession, + forkSession, + prompt, + setSessionMode, +} from "@/shared/api/acpApi"; +import { + hidePrivateSessionNotifications, + interceptSessionNotifications, +} from "@/shared/api/acpConnection"; + +export interface PrivateVoiceDraft { + take: () => Promise; + dispose: () => void; +} + +/** No main-session run, mode, permission or notification state is changed. */ +export function startPrivateVoiceDraft( + sourceSessionId: string, + workingDir: string, + text: string, +): PrivateVoiceDraft { + let stopped = false; + let forkId: string | undefined; + let stopIntercepting: (() => void) | undefined; + let setupTimer: ReturnType | undefined; + const cleanupFork = (sessionId: string) => { + hidePrivateSessionNotifications(sessionId); + // Neither cancellation nor deletion is trusted to settle. They cannot hold + // the final queue, and neither ever targets the source session. + void cancelSession(sessionId).catch(() => undefined); + void deleteSession(sessionId).catch(() => undefined); + }; + const dispose = () => { + if (stopped) return; + stopped = true; + clearTimeout(setupTimer); + clearTimeout(lifetimeTimer); + stopIntercepting?.(); + if (forkId) cleanupFork(forkId); + }; + const lifetimeTimer = setTimeout(dispose, 15_000); + setupTimer = setTimeout(dispose, 2_000); + const result = (async () => { + const fork = await forkSession(sourceSessionId, workingDir); + forkId = fork.sessionId; + hidePrivateSessionNotifications(forkId); + if (stopped) { + cleanupFork(forkId); + return null; + } + // Fail closed. Backend chat mode skips ALL tool_requests, including tools + // that bypass the permission callback. Prompt must never precede this ack. + await setSessionMode(forkId, "chat"); + if (stopped) return null; + clearTimeout(setupTimer); + const chunks: string[] = []; + stopIntercepting = interceptSessionNotifications((notification) => { + if (notification.sessionId !== forkId) return false; + const { update } = notification; + if ( + !stopped && + update.sessionUpdate === "agent_message_chunk" && + update.content.type === "text" + ) + chunks.push(update.content.text); + return true; + }); + await prompt(forkId, [{ type: "text", text }]); + return stopped ? null : chunks.join("").trim() || null; + })() + .catch(() => null) + .finally(dispose); + return { + dispose, + async take() { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + result, + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), 750); + }), + ]); + } finally { + clearTimeout(timer); + dispose(); + } + }, + }; +} diff --git a/src/features/voice-conversation/stores/voiceConversationStore.test.ts b/src/features/voice-conversation/stores/voiceConversationStore.test.ts index 4fbe73445..1096a190b 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.test.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.test.ts @@ -267,53 +267,6 @@ describe("voice conversation store lifecycle ordering", () => { unsubscribe(); }); - it("holds speculative speech during retries but rejects it on terminal delivery failure", async () => { - const module = await import("./voiceConversationStore"); - const unsubscribe = module.subscribeToVoiceConversationEvents(() => - Promise.reject(new Error("chat unavailable")), - ); - await module.useVoiceConversationStore.getState().init(); - const transcript = { - sessionId: "session-1", - lifecycleId: "lifecycle-1", - revision: 1, - id: "failed-final", - text: "Cannot deliver", - deliveryAttempts: 0, - }; - const key = ["session-1", "lifecycle-1", "1", "failed-final"].join("\0"); - module.useVoiceConversationStore.setState({ - earlyTranscriptPending: key, - earlyTranscriptResolution: null, - latestFinalizedTranscriptKey: "prior-transcript", - }); - mocks.drain.mockResolvedValue([transcript]); - await expect( - module.useVoiceConversationStore - .getState() - .drainPendingTranscripts("session-1"), - ).rejects.toThrow("delivery was rejected"); - expect( - module.useVoiceConversationStore.getState().earlyTranscriptPending, - ).toBe(key); - mocks.reject.mockResolvedValueOnce({ attempts: 3, terminal: true }); - await expect( - module.useVoiceConversationStore - .getState() - .drainPendingTranscripts("session-1"), - ).rejects.toThrow("delivery was rejected"); - expect(module.useVoiceConversationStore.getState()).toMatchObject({ - earlyTranscriptPending: null, - earlyTranscriptResolution: { - key: `${key}:early`, - finalKey: key, - accepted: false, - }, - latestFinalizedTranscriptKey: "prior-transcript", - }); - unsubscribe(); - }); - it("removes terminally rejected transcripts from pending rollback chains", async () => { mocks.reject.mockResolvedValue({ attempts: 3, terminal: true }); const firstDelivery = deferred(); diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index dced41a7b..35ac0f11a 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -58,15 +58,6 @@ interface VoiceConversationStore { hydrated: boolean; requestedStartSessionId: string | null; latestFinalizedTranscriptKey: string | null; - // Internal speech gate, not a visual state. Speculative replies cannot be - // spoken until their final transcript has been reconciled. - earlyTranscriptPending: string | null; - earlyTranscriptResolution: { - key: string; - finalKey: string; - accepted: boolean; - terminal?: boolean; - } | null; init: () => Promise; refreshStatus: () => Promise; requestStart: (sessionId: string) => void; @@ -246,18 +237,6 @@ async function deliverTranscriptOnce( ...(state.latestFinalizedTranscriptKey === finalizedKey ? { latestFinalizedTranscriptKey: priorKey } : {}), - ...(state.earlyTranscriptPending === finalizedKey || - state.earlyTranscriptResolution?.finalKey === finalizedKey - ? { - earlyTranscriptPending: null, - earlyTranscriptResolution: { - key: `${finalizedKey}:early`, - finalKey: finalizedKey, - accepted: false, - terminal: true, - }, - } - : {}), })); } } @@ -332,8 +311,6 @@ export const useVoiceConversationStore = create( hydrated: false, requestedStartSessionId: null, latestFinalizedTranscriptKey: null, - earlyTranscriptPending: null, - earlyTranscriptResolution: null, requestStart: (sessionId) => set({ requestedStartSessionId: sessionId }), clearRequestedStart: (sessionId) => @@ -464,8 +441,7 @@ export const useVoiceConversationStore = create( revision: event.revision, }, uiState: "listening", - earlyTranscriptPending: null, - earlyTranscriptResolution: null, + microphoneMuted: false, latestFinalizedTranscriptKey: null, error: null, diff --git a/src/shared/api/__tests__/acp.test.ts b/src/shared/api/__tests__/acp.test.ts index d0013d482..9b645d0d1 100644 --- a/src/shared/api/__tests__/acp.test.ts +++ b/src/shared/api/__tests__/acp.test.ts @@ -177,46 +177,6 @@ describe("acpSteerMessage", () => { }); describe("acpSendMessage", () => { - it.each([ - false, - true, - ])("uses real assistant-only audience for provisional input (steer: %s)", async (steer) => { - const registry = await import("../acpSessionRegistry"); - const { acpSendMessage, acpSteerMessage } = await import("../acp"); - registry.registerPreparedSession( - "early-audience", - "goose", - "/tmp/project", - "test-model", - ); - const options = { - assistantOnlyInput: true, - assistantPrompt: "Provisional speech.", - }; - if (steer) - await acpSteerMessage("early-audience", "run-1", "early words", options); - else await acpSendMessage("early-audience", "early words", options); - const provisional = (steer ? mockSteerSession : mockPrompt).mock.calls.at( - -1, - )?.[1]; - expect(provisional).toEqual([ - { - type: "text", - text: "Provisional speech.", - annotations: { audience: ["assistant"] }, - }, - { - type: "text", - text: "early words", - annotations: { audience: ["assistant"] }, - }, - ]); - if (steer) await acpSteerMessage("early-audience", "run-1", "final words"); - else await acpSendMessage("early-audience", "final words"); - expect( - (steer ? mockSteerSession : mockPrompt).mock.calls.at(-1)?.[1], - ).toEqual([{ type: "text", text: "final words" }]); - }); beforeEach(() => { vi.clearAllMocks(); vi.resetModules(); diff --git a/src/shared/api/__tests__/acpBackends.test.ts b/src/shared/api/__tests__/acpBackends.test.ts index c08de52a5..32c5c4f7a 100644 --- a/src/shared/api/__tests__/acpBackends.test.ts +++ b/src/shared/api/__tests__/acpBackends.test.ts @@ -924,6 +924,31 @@ describe("inbound session id translation", () => { stop(); }); + it("fences private notifications and permissions before the normal handler", async () => { + const conn = await importConnection(); + const received = vi.fn(); + const permissions = vi.fn().mockResolvedValue({ + outcome: { outcome: "selected", optionId: "approve" }, + }); + conn.setNotificationHandler({ handleSessionNotification: received }); + conn.setPermissionHandler(permissions); + await conn.getBackendConnection(LOCAL_BACKEND_ID).getClient(); + const callbacks = await latestCallbacks(); + conn.hidePrivateSessionNotifications("private-voice"); + await callbacks.sessionUpdate?.(sessionUpdatePayload("private-voice")); + expect(received).not.toHaveBeenCalled(); + await expect( + callbacks.requestPermission({ + sessionId: "private-voice", + toolCall: { toolCallId: "write", title: "write_file" }, + options: [{ optionId: "approve", name: "Allow", kind: "allow_once" }], + }), + ).resolves.toEqual({ outcome: { outcome: "cancelled" } }); + expect(permissions).not.toHaveBeenCalled(); + await callbacks.sessionUpdate?.(sessionUpdatePayload("ordinary")); + expect(received).toHaveBeenCalledOnce(); + }); + it("rewrites remote notification session ids to composite ids", async () => { const conn = await importConnection(); const received: SessionNotification[] = []; diff --git a/src/shared/api/acp.ts b/src/shared/api/acp.ts index 3e9d0b313..c7c8c156f 100644 --- a/src/shared/api/acp.ts +++ b/src/shared/api/acp.ts @@ -55,8 +55,6 @@ export interface AcpProvider { export interface AcpSendMessageOptions { systemPrompt?: string; assistantPrompt?: string; - /** Internal provisional context, persisted for the assistant only. */ - assistantOnlyInput?: boolean; personaId?: string; personaName?: string; goose?: Record; @@ -168,7 +166,6 @@ async function acpSendMessageNow( const { systemPrompt, assistantPrompt, - assistantOnlyInput, personaId, personaName, goose, @@ -249,13 +246,7 @@ async function acpSendMessageNow( annotations: { audience: ["assistant"] }, }); } - content.push({ - type: "text", - text: prompt, - ...(assistantOnlyInput - ? { annotations: { audience: ["assistant" as const] } } - : {}), - }); + content.push({ type: "text", text: prompt }); if (images) { for (const [data, mimeType] of images) { content.push({ type: "image", data, mimeType } as ContentBlock); @@ -311,11 +302,11 @@ export async function acpSteerMessage( prompt: string, options: Pick< AcpSendMessageOptions, - "assistantPrompt" | "assistantOnlyInput" | "goose" | "images" + "assistantPrompt" | "goose" | "images" > = {}, ): Promise { sessionRegistry.requireSessionInvocationSelection(sessionId); - const { assistantPrompt, assistantOnlyInput, goose, images } = options; + const { assistantPrompt, goose, images } = options; const content: ContentBlock[] = []; const assistantText = assistantPrompt?.trim(); if (assistantText) { @@ -325,13 +316,7 @@ export async function acpSteerMessage( annotations: { audience: ["assistant"] }, }); } - content.push({ - type: "text", - text: prompt, - ...(assistantOnlyInput - ? { annotations: { audience: ["assistant" as const] } } - : {}), - }); + content.push({ type: "text", text: prompt }); if (images) { for (const [data, mimeType] of images) { content.push({ type: "image", data, mimeType } as ContentBlock); diff --git a/src/shared/api/acpApi.ts b/src/shared/api/acpApi.ts index 521c90dee..8bf6b4f6f 100644 --- a/src/shared/api/acpApi.ts +++ b/src/shared/api/acpApi.ts @@ -515,6 +515,18 @@ export async function renameSession( }); } +/** Raw ACP mode change. Callers must await success before private inference. */ +export async function setSessionMode( + sessionId: string, + modeId: string, +): Promise { + const client = await getClientForSession(sessionId); + await client.setSessionMode({ + sessionId: getWireSessionId(sessionId), + modeId, + }); +} + export async function cancelSession(sessionId: string): Promise { const client = await getClientForSession(sessionId); await client.cancel({ sessionId: getWireSessionId(sessionId) }); diff --git a/src/shared/api/acpConnection.ts b/src/shared/api/acpConnection.ts index 9f172cdc9..ee7dd0911 100644 --- a/src/shared/api/acpConnection.ts +++ b/src/shared/api/acpConnection.ts @@ -26,6 +26,13 @@ import { import { createWebSocketStream } from "./createWebSocketStream"; import { perfLog } from "@/shared/lib/perfLog"; +const privateSessionIds = new Set(); + +/** Keep a text-free fence even after cancellation: transports may emit late chunks. */ +export function hidePrivateSessionNotifications(sessionId: string): void { + privateSessionIds.add(sessionId); +} + let notificationHandler: AcpNotificationHandler | null = null; const sessionNotificationInterceptors = new Set(); @@ -101,6 +108,7 @@ function createClientCallbacks( }; if (!isCurrent()) return cancelled; const request = toRendererSessionId(args); + if (privateSessionIds.has(request.sessionId)) return cancelled; if (permissionHandler) { // Cancel immediately on detachment even if a handler never settles. // The signal also retires this generation's pending confirmation UI. @@ -142,6 +150,7 @@ function createClientCallbacks( return; } } + if (privateSessionIds.has(notification.sessionId)) return; if (notificationHandler) { await notificationHandler.handleSessionNotification(notification); } diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index 72f9aac05..b97e58336 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -25,6 +25,7 @@ export function buildInitScript(options?: { homeDir?: string; modelId?: string; holdAcpPrompts?: boolean; + privateVoiceDraftScenario?: "complete" | "stalled" | "mode-rejected"; personas?: unknown[]; skills?: unknown[]; projects?: unknown[]; @@ -121,6 +122,7 @@ export function buildInitScript(options?: { return ` (() => { + const PRIVATE_VOICE_DRAFT = ${JSON.stringify(options?.privateVoiceDraftScenario ?? null)}; const HOLD_ACP_PROMPTS = ${options?.holdAcpPrompts ?? false}; const ACP_REQUESTS = []; const PENDING_ACP_PROMPTS = new Map(); @@ -179,6 +181,7 @@ export function buildInitScript(options?: { lastMessageAt: session.lastMessageAt, workingDir: session.workingDir, replayText: session.replayText, + cwd: session.workingDir ?? "/tmp/alpha", providerId: session.providerId ?? "goose", modelId: session.modelId ?? null, })); @@ -427,6 +430,7 @@ export function buildInitScript(options?: { sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, + cwd: session.cwd, _meta: { messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, @@ -453,6 +457,7 @@ export function buildInitScript(options?: { providerId: source?.providerId ?? "goose", modelId: source?.modelId ?? null, conversationBefore: message.params?._meta?.conversationBefore, + mode: null, }; ACP_SESSIONS.unshift(copy); return jsonRpcResult(message.id, { @@ -482,7 +487,10 @@ export function buildInitScript(options?: { update: { sessionUpdate: "user_message_chunk", content: { type: "text", text: session.replayText } }, }); } - return jsonRpcResult(message.id, {}); + return jsonRpcResult(message.id, { configOptions: [ + { id: "provider", kind: { type: "select", currentValue: session?.providerId ?? "openai", options: [] } }, + { id: "model", category: "model", kind: { type: "select", currentValue: session?.modelId ?? "gpt-4.1", options: [] } }, + ] }); } case "session/set_config_option": { const session = findSession(message.params?.sessionId); @@ -498,6 +506,17 @@ export function buildInitScript(options?: { } return jsonRpcResult(message.id, {}); } + case "session/set_mode": { + const session = findSession(message.params?.sessionId); + if (PRIVATE_VOICE_DRAFT === "mode-rejected") return { jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Chat mode unavailable" } }; + if (session) session.mode = message.params.modeId; + return jsonRpcResult(message.id, {}); + } + case "session/delete": { + const index = ACP_SESSIONS.findIndex((session) => session.sessionId === message.params?.sessionId); + if (index >= 0) ACP_SESSIONS.splice(index, 1); + return jsonRpcResult(message.id, {}); + } case "_goose/unstable/session/steer": return jsonRpcResult(message.id, { runId: message.params.expectedRunId, @@ -705,6 +724,19 @@ export function buildInitScript(options?: { send(raw) { const message = JSON.parse(raw); ACP_REQUESTS.push(clone(message)); + // Semantic model boundary: private inference is only accepted after + // a successful chat-mode transition, not merely an audience hint. + if (PRIVATE_VOICE_DRAFT && message.method === "session/prompt" && message.params.sessionId.startsWith("session-fork-")) { + const session = findSession(message.params.sessionId); + if (session?.mode !== "chat") throw new Error("Private model invoked with tool authority"); + if (PRIVATE_VOICE_DRAFT === "complete") { + queueMicrotask(() => { + window.__GOOSE_E2E__.emitAcpNotification({ sessionId: session.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "PRIVATE DRAFT: I would set the temperature to 10 degrees." } } }); + this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(jsonRpcResult(message.id, { stopReason: "end_turn" })) })); + }); + return; + } + } if (HOLD_ACP_PROMPTS && message.method === "session/prompt") { PENDING_ACP_PROMPTS.set(message.id, { socket: this, sessionId: message.params.sessionId }); return; @@ -959,11 +991,13 @@ export function buildInitScript(options?: { sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, + cwd: session.cwd, messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, providerId: session.providerId, modelId: session.modelId, conversationBefore: session.conversationBefore, + cwd: session.workingDir ?? "/tmp/alpha", })), ); case "create_session": diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index c6cc321ae..f5e193a11 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -53,7 +53,7 @@ test("queues assistant text through native playback", async ({ harness.emitTauriEvent("voice-conversation:event", { type: "startup", sessionId, - ownerWindowLabel: "voice-e2e-owner", + ownerWindowLabel: "main", line: "Voice conversation ready", revision: 2, }); @@ -120,6 +120,8 @@ test("queues assistant text through native playback", async ({ type AcpRequest = { method: string; params?: { + sessionId?: string; + modeId?: string; prompt?: { text?: string; annotations?: { audience?: string[] } }[]; _meta?: { goose?: { voiceUtteranceId?: string } }; }; @@ -215,196 +217,218 @@ const utterance = { text: "Set the temperature to 10 degrees.", }; -base.describe("early voice delivery", () => { - base.beforeEach(async ({ page }) => { - await page.addInitScript({ - content: buildInitScript({ - holdAcpPrompts: true, - sessions: [ - { - sessionId: SESSION_ID, - title: "Early voice", - messageCount: 1, - providerId: "openai", - modelId: "gpt-4.1", +for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { + base.describe(`isolated early voice (${scenario})`, () => { + base.beforeEach(async ({ page }) => { + await page.addInitScript({ + content: buildInitScript({ + holdAcpPrompts: true, + privateVoiceDraftScenario: scenario, + sessions: [ + { + sessionId: SESSION_ID, + title: "Early voice", + messageCount: 1, + providerId: "openai", + modelId: "gpt-4.1", + workingDir: "/workspace", + }, + ], + voiceConversationStatus: { + available: true, + unavailableReason: null, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: 1, }, - ], - voiceConversationStatus: { - available: true, - unavailableReason: null, - lifecycle: "stopped", - sessionId: null, - ownerWindowLabel: null, - revision: 1, - }, - enabledExperiments: ["voice-conversation"], - }), - }); - await page.goto("/"); - await page.getByText("Early voice", { exact: true }).click(); - await expect(page.getByRole("textbox").first()).toBeVisible(); - await page.evaluate( - () => - new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(() => resolve())), - ), - ); - await nativeEvent(page, { - type: "startup", - ownerWindowLabel: "main", - line: "Ready", - }); - await assistantOutput(page, "The reply being interrupted.", "prior-run"); - await expect - .poll(() => spoken(page)) - .toEqual(["The reply being interrupted."]); - await nativeEvent(page, { type: "activity", activity: "user-speaking" }); - await page.evaluate(() => { - const harness = ( - window as typeof window & { __GOOSE_E2E__: VoiceHarness } - ).__GOOSE_E2E__; - harness.emitTauriEvent("pocket-voice:stream-event", { - streamId: harness.pocketVoiceStreamIds()[0], - state: "interrupted", - error: null, - delivery: { segments: [] }, - }); - }); - await page.evaluate((sessionId) => { - ( - window as typeof window & { __GOOSE_E2E__: VoiceHarness } - ).__GOOSE_E2E__.emitAcpNotification({ - sessionId, - update: { - sessionUpdate: "session_info_update", - _meta: { goose: { activeRunId: null } }, - }, - }); - }, SESSION_ID); - await nativeEvent(page, { type: "interimUser", ...utterance }); - await nativeEvent(page, { type: "activity", activity: "user-idle" }); - await expect.poll(() => requests(page)).toHaveLength(1); - const early = (await requests(page))[0]; - expect(early.method).toBe("session/prompt"); - expect( - early.params?.prompt?.find((block) => block.text === utterance.text), - ).toMatchObject({ annotations: { audience: ["assistant"] } }); - await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( - 0, - ); - }); - - base( - "unchanged final is visible once and releases withheld assistant speech", - async ({ page }) => { - const answer = "The unchanged provisional answer can now be spoken."; - await assistantOutput(page, answer, "early-run"); - await nativeEvent(page, { type: "microphoneMute", muted: true }); - await expect - .poll(() => spoken(page)) - .toEqual(["The reply being interrupted."]); - await nativeEvent(page, { - type: "user", - ...utterance, - deliveryAttempts: 0, + enabledExperiments: ["voice-conversation"], + }), }); - await expect.poll(() => requests(page)).toHaveLength(2); - expect((await requests(page))[1].method).toBe( - "_goose/unstable/session/steer", - ); - expect( - (await requests(page))[1].params?.prompt?.find( - (block) => block.text === utterance.text, - )?.annotations, - ).toBeUndefined(); - await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( - 1, + await page.goto("/"); + await page.getByText("Early voice", { exact: true }).click(); + await expect(page.getByRole("textbox").first()).toBeVisible(); + await page.evaluate( + () => + new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ), ); - await expect - .poll(() => spoken(page)) - .toEqual(["The reply being interrupted.", answer]); await nativeEvent(page, { - type: "user", - ...utterance, - deliveryAttempts: 0, + type: "startup", + ownerWindowLabel: "main", + line: "Ready", }); - await completePrompts(page); - await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( - 1, + }); + + for (const busy of [false, true]) + base( + `final has sole tool authority, bounded delivery and later utterance (busy: ${busy})`, + async ({ page }) => { + if (busy) + await assistantOutput( + page, + "An unrelated running answer.", + "unrelated-run", + ); + await nativeEvent(page, { type: "interimUser", ...utterance }); + const allRequests = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.acpRequests(), + ); + await expect + .poll(async () => + (await allRequests()).filter( + (r) => r.method === "session/set_mode", + ), + ) + .toHaveLength(1); + const mode = (await allRequests()).find( + (r) => r.method === "session/set_mode", + ); + const privateId = mode?.params?.sessionId; + if (!privateId) throw new Error("Missing private fork"); + expect(privateId).not.toBe(SESSION_ID); + expect(mode?.params?.modeId).toBe("chat"); + if (scenario !== "mode-rejected") { + await expect.poll(() => requests(page)).toHaveLength(1); + expect((await requests(page))[0].params?.sessionId).toBe(privateId); + const wire = await allRequests(); + expect( + wire.findIndex((r) => r.method === "session/set_mode"), + ).toBeLessThan( + wire.findIndex((r) => r.method === "session/prompt"), + ); + } else expect(await requests(page)).toHaveLength(0); + await expect( + page.getByText(utterance.text, { exact: true }), + ).toHaveCount(0); + await expect( + page.getByText("PRIVATE DRAFT:", { exact: false }), + ).toHaveCount(0); + const finalText = busy + ? utterance.text + : "Set the temperature to -10 degrees."; + const started = Date.now(); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + const mainRequests = async () => + (await requests(page)).filter( + (r) => r.params?.sessionId === SESSION_ID, + ); + await expect + .poll(mainRequests, { timeout: 1800, intervals: [20, 50] }) + .toHaveLength(1); + expect(Date.now() - started).toBeLessThan(1800); + const final = (await mainRequests())[0]; + expect(final.method).toBe( + busy ? "_goose/unstable/session/steer" : "session/prompt", + ); + expect( + final.params?.prompt?.find((b) => b.text === finalText) + ?.annotations, + ).toBeUndefined(); + const context = + final.params?.prompt + ?.filter((b) => b.annotations?.audience?.includes("assistant")) + .map((b) => b.text) + .join("\n") ?? ""; + if (scenario === "complete") { + expect(context).toContain("PRIVATE DRAFT"); + expect(context).toContain("perform any requested actions normally"); + } else expect(context).not.toContain("PRIVATE DRAFT"); + expect(context).not.toContain("do not repeat"); + await expect(page.getByText(finalText, { exact: true })).toHaveCount( + 1, + ); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + expect(await mainRequests()).toHaveLength(1); + const cancellations = (await allRequests()).filter( + (r) => r.method === "session/cancel", + ); + if (scenario !== "mode-rejected") + expect(cancellations.length).toBeGreaterThan(0); + expect( + cancellations.every((r) => r.params?.sessionId === privateId), + ).toBe(true); + // Late private output is fenced even after bounded cancellation/cleanup. + await page.evaluate( + (sessionId) => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "LATE PRIVATE OUTPUT" }, + }, + }), + privateId, + ); + await expect( + page.getByText("LATE PRIVATE OUTPUT", { exact: false }), + ).toHaveCount(0); + expect( + (await spoken(page)).every((text) => !text.includes("PRIVATE")), + ).toBe(true); + if (!busy) + await assistantOutput( + page, + "The final action has been performed.", + "final-run", + ); + await nativeEvent(page, { + type: "user", + ...utterance, + id: "utterance-2", + text: "Now report the result.", + deliveryAttempts: 0, + }); + await expect.poll(mainRequests).toHaveLength(2); + expect((await mainRequests())[1].method).toBe( + "_goose/unstable/session/steer", + ); + await completePrompts(page); + }, ); - expect(await requests(page)).toHaveLength(2); - expect(await spoken(page)).toEqual([ - "The reply being interrupted.", - answer, - ]); - }, - ); - for (const finalText of [ - "Set the temperature to -10 degrees.", - "set the temperature to 10 degrees.", - ]) base( - `changed final (${finalText}) never speaks the old answer, including late old output`, + "recognition ending without final permits another private utterance", async ({ page }) => { - const oldAnswer = - "This obsolete provisional answer must never be spoken."; - await assistantOutput(page, oldAnswer, "obsolete-run"); - await nativeEvent(page, { type: "microphoneMute", muted: true }); + await nativeEvent(page, { type: "interimUser", ...utterance }); + const forks = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__ + .acpRequests() + .filter((r) => r.method === "session/fork"), + ); + await expect.poll(forks).toHaveLength(1); + await nativeEvent(page, { type: "recognitionPending", pending: false }); await nativeEvent(page, { - type: "user", + type: "interimUser", ...utterance, - text: finalText, - deliveryAttempts: 0, + id: "utterance-2", }); - await assistantOutput( - page, - " Late obsolete output must also stay unspoken.", - "obsolete-run", - ); - expect(await requests(page)).toHaveLength(1); - expect(await spoken(page)).toEqual(["The reply being interrupted."]); - await completePrompts(page); - await expect.poll(() => requests(page)).toHaveLength(2); - expect((await requests(page))[1].method).toBe("session/prompt"); - await expect(page.getByText(finalText, { exact: true })).toHaveCount(1); - await expect( - page.getByText(utterance.text, { exact: true }), - ).toHaveCount(0); - const correctedAnswer = - "Only the corrected final answer should be spoken."; - await assistantOutput(page, correctedAnswer, "corrected-run"); - await completePrompts(page); - await expect - .poll(() => spoken(page)) - .toEqual(["The reply being interrupted.", correctedAnswer]); + await expect.poll(forks).toHaveLength(2); + expect( + (await requests(page)).every( + (r) => r.params?.sessionId !== SESSION_ID, + ), + ).toBe(true); }, ); - - base( - "recognition ending without a final discards speculative speech and permits the next utterance", - async ({ page }) => { - await assistantOutput( - page, - "An abandoned answer must remain unspoken.", - "abandoned-run", - ); - await nativeEvent(page, { type: "recognitionPending", pending: false }); - await completePrompts(page); - await nativeEvent(page, { - type: "interimUser", - ...utterance, - id: "utterance-2", - text: "The next utterance.", - }); - await expect.poll(() => requests(page)).toHaveLength(2); - expect(await spoken(page)).toEqual(["The reply being interrupted."]); - await expect(page.getByText(utterance.text, { exact: true })).toHaveCount( - 0, - ); - await nativeEvent(page, { type: "cleanShutdown" }); - await completePrompts(page); - expect(await spoken(page)).toEqual(["The reply being interrupted."]); - }, - ); -}); + }); +} From c80376062edbbf12603c4b9a874dfe7d1636c0f8 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 17:45:50 -0400 Subject: [PATCH 11/19] feat(voice): add default-on experimental early delivery setting Signed-off-by: John Matthew Tennant --- .../useVoiceConversationController.test.ts | 81 ++++++++++ .../hooks/useVoiceConversationController.ts | 13 +- .../lib/voiceEarlyDeliveryPreference.test.ts | 42 +++++ .../lib/voiceEarlyDeliveryPreference.ts | 146 ++++++++++++++++++ .../ui/VoiceSettings.test.tsx | 26 ++++ .../voice-conversation/ui/VoiceSettings.tsx | 20 +++ src/shared/i18n/locales/en/settings.json | 2 + src/shared/i18n/locales/es/settings.json | 2 + tests/e2e/voice-conversation.spec.ts | 118 ++++++++++++++ 9 files changed, 449 insertions(+), 1 deletion(-) create mode 100644 src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts create mode 100644 src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 6752807b8..e372fedae 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,8 +3,17 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import { VoiceMicrophoneCaptureError } from "../api/voiceConversation"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { setVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { setStatusSoundPreference } from "../lib/statusSoundPreference"; +const privateDraftMocks = vi.hoisted(() => ({ + start: vi.fn(), +})); +vi.mock("../lib/privateVoiceDraft", () => ({ + startPrivateVoiceDraft: privateDraftMocks.start, +})); + const nativeAssistantSpeechMocks = vi.hoisted(() => ({ capture: vi.fn(() => []), start: vi.fn(), @@ -319,6 +328,78 @@ describe("voice transcript delivery coordination", () => { useChatStore.setState({ messagesBySession: {}, sessionStateById: {} }); }); + it("gates preparation, disposes outstanding drafts on disable, and still delivers finals", async () => { + const dispose = vi.fn(); + const take = vi.fn().mockResolvedValue("private draft"); + privateDraftMocks.start.mockReset(); + privateDraftMocks.start.mockReturnValue({ dispose, take }); + const onSend = vi.fn().mockResolvedValue(true); + useChatSessionStore.setState({ + sessions: [{ id: "session-1", workingDir: "/project" }] as never, + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 42, + }, + microphoneMuted: false, + uiState: "listening", + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + }); + const control = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend, + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + const event = { + sessionId: "session-1", + lifecycleId: "early-toggle", + id: "toggle-utterance", + text: "partial", + revision: 42, + deliveryAttempts: 0, + }; + await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); + expect(privateDraftMocks.start).toHaveBeenCalledOnce(); + act(() => setVoiceEarlyDeliveryPreference({ enabled: false })); + expect(dispose).toHaveBeenCalledOnce(); + await voiceStoreMocks.subscriber?.({ + ...event, + id: "disabled-utterance", + type: "interimUser", + }); + expect(privateDraftMocks.start).toHaveBeenCalledOnce(); + await act(async () => { + await voiceStoreMocks.subscriber?.({ + ...event, + text: "authoritative final", + type: "user", + }); + }); + expect(take).not.toHaveBeenCalled(); + expect(onSend).toHaveBeenCalledWith( + "authoritative final", + undefined, + undefined, + expect.objectContaining({ + displayText: "authoritative final", + assistantPrompt: "", + }), + ); + control.unmount(); + }); + it("applies settings changes to an active chained runtime", async () => { useVoiceConversationStore.setState({ status: { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 49434cd01..37fac1d9d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -11,6 +11,10 @@ import { startPrivateVoiceDraft, type PrivateVoiceDraft, } from "../lib/privateVoiceDraft"; +import { + getVoiceEarlyDeliveryPreference, + subscribeToVoiceEarlyDeliveryPreference, +} from "../lib/voiceEarlyDeliveryPreference"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; import { createSystemNotificationMessage } from "@/shared/types/messages"; @@ -559,6 +563,12 @@ function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; observeChainedVoiceStatus(); + subscribeToVoiceEarlyDeliveryPreference(() => { + if (getVoiceEarlyDeliveryPreference().enabled) return; + for (const draft of earlyDeliveries.values()) draft.dispose(); + earlyDeliveries.clear(); + finalTranscriptsSeen.clear(); + }); useVoiceConversationStore.subscribe((state, previous) => { if ( state.status.revision === previous.status.revision && @@ -617,6 +627,7 @@ function ensureVoiceEventDeliveryInitialized() { const shouldNotifyFailure = event.deliveryAttempts === 0; if (isEarly) { if ( + !getVoiceEarlyDeliveryPreference().enabled || !earlyEventIsCurrent(event) || earlyDeliveries.has(eventKey) || activeRouteIsBlocked(event.sessionId) || @@ -696,7 +707,7 @@ function ensureVoiceEventDeliveryInitialized() { ...sendOptions, assistantPrompt: [ playbackNotice, - draft + draft && getVoiceEarlyDeliveryPreference().enabled ? `Private provisional DRAFT (not spoken, not executed, may be wrong):\n${draft}\n\nThe user transcript in this turn is authoritative. Answer it and perform any requested actions normally. Use the draft only if it is relevant to the final transcript.` : null, ] diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts new file mode 100644 index 000000000..baa18e3a4 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts @@ -0,0 +1,42 @@ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + getVoiceEarlyDeliveryPreference, + setVoiceEarlyDeliveryPreference, + useVoiceEarlyDeliveryPreference, +} from "./voiceEarlyDeliveryPreference"; + +const key = "goose:voice-early-delivery-preference"; + +describe("voice early delivery preference", () => { + beforeEach(() => window.localStorage.clear()); + + it("defaults on and preserves saved off", () => { + expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: true }); + setVoiceEarlyDeliveryPreference({ enabled: false }); + expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: false }); + expect(JSON.parse(window.localStorage.getItem(key) ?? "null")).toEqual({ + enabled: false, + }); + }); + + it.each([ + "invalid json", + "{}", + '{"enabled":"false"}', + ])("defaults on for invalid storage: %s", (raw) => { + window.localStorage.setItem(key, raw); + expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: true }); + }); + + it("updates same-window and cross-window subscribers", () => { + const { result } = renderHook(() => useVoiceEarlyDeliveryPreference()); + act(() => result.current.setEnabled(false)); + expect(result.current.enabled).toBe(false); + act(() => { + window.localStorage.setItem(key, JSON.stringify({ enabled: true })); + window.dispatchEvent(new StorageEvent("storage", { key })); + }); + expect(result.current.enabled).toBe(true); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts new file mode 100644 index 000000000..1f6af5d09 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts @@ -0,0 +1,146 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export interface VoiceEarlyDeliveryPreference { + enabled: boolean; +} + +const STORAGE_KEY = "goose:voice-early-delivery-preference"; +const CHANGED_EVENT = "goose:voice-early-delivery-preference-changed"; +const DEFAULT_PREFERENCE: VoiceEarlyDeliveryPreference = { + enabled: true, +}; +const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); +let volatilePreference: + | { + preference: VoiceEarlyDeliveryPreference; + storageValueBeforeWrite: string | null | undefined; + } + | undefined; +let removeVolatileStorageListener: (() => void) | undefined; + +function normalize(value: unknown): VoiceEarlyDeliveryPreference { + if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; + const candidate = value as Partial; + return { + enabled: + typeof candidate.enabled === "boolean" + ? candidate.enabled + : DEFAULT_PREFERENCE.enabled, + }; +} + +export function getDefaultVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + return DEFAULT_PREFERENCE; +} + +export function getVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + if (typeof window === "undefined") return DEFAULT_PREFERENCE; + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + if (volatilePreference) { + if ( + volatilePreference.storageValueBeforeWrite === undefined || + raw === volatilePreference.storageValueBeforeWrite + ) { + return volatilePreference.preference; + } + clearVolatilePreference(); + } + return raw ? normalize(JSON.parse(raw)) : DEFAULT_PREFERENCE; + } catch { + return volatilePreference?.preference ?? DEFAULT_PREFERENCE; + } +} + +function getSnapshot(): string { + return JSON.stringify(getVoiceEarlyDeliveryPreference()); +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function clearVolatilePreference() { + volatilePreference = undefined; + removeVolatileStorageListener?.(); + removeVolatileStorageListener = undefined; +} + +function retainVolatilePreference( + preference: VoiceEarlyDeliveryPreference, + storageValueBeforeWrite: string | null | undefined, +) { + volatilePreference = { preference, storageValueBeforeWrite }; + if (removeVolatileStorageListener || typeof window === "undefined") return; + const handleStorage = (event: StorageEvent) => { + if (event.key !== STORAGE_KEY && event.key !== null) return; + clearVolatilePreference(); + notify(); + }; + window.addEventListener("storage", handleStorage); + removeVolatileStorageListener = () => { + window.removeEventListener("storage", handleStorage); + }; +} + +export function subscribeToVoiceEarlyDeliveryPreference(listener: () => void) { + if (typeof window === "undefined") return () => {}; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) { + clearVolatilePreference(); + notify(); + } + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setVoiceEarlyDeliveryPreference( + preference: VoiceEarlyDeliveryPreference, +): void { + if (typeof window === "undefined") return; + const value = normalize(preference); + let storageValueBeforeWrite: string | null | undefined; + try { + storageValueBeforeWrite = window.localStorage.getItem(STORAGE_KEY); + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); + clearVolatilePreference(); + } catch { + // Keep the current in-memory renderer usable when storage is unavailable. + retainVolatilePreference(value, storageValueBeforeWrite); + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: value })); +} + +export function useVoiceEarlyDeliveryPreference() { + const snapshot = useSyncExternalStore( + subscribeToVoiceEarlyDeliveryPreference, + getSnapshot, + () => DEFAULT_SNAPSHOT, + ); + const preference = normalize(JSON.parse(snapshot)); + const setEnabled = useCallback((enabled: boolean) => { + setVoiceEarlyDeliveryPreference({ + ...getVoiceEarlyDeliveryPreference(), + enabled, + }); + }, []); + return { ...preference, setEnabled }; +} diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 5edf6e57f..c27921a03 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -9,6 +9,7 @@ import type { MacSpeechSetup } from "../hooks/useMacSpeechSetup"; import type { SiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import type { VoiceOutputBackend } from "../lib/voiceOutputPreference"; +import { setVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { VoiceSettings } from "./VoiceSettings"; if (!HTMLElement.prototype.hasPointerCapture) { @@ -277,7 +278,32 @@ function siriSetup(): SiriVoiceSetup { } describe("VoiceSettings", () => { + it("defaults early delivery on and persists an explicit off choice", async () => { + const user = userEvent.setup(); + const view = renderWithProviders(); + const toggle = screen.getByRole("switch", { + name: "Early delivery (experimental)", + }); + expect(toggle).toBeChecked(); + await user.click(toggle); + expect(toggle).not.toBeChecked(); + view.unmount(); + renderWithProviders(); + expect( + screen.getByRole("switch", { name: "Early delivery (experimental)" }), + ).not.toBeChecked(); + }); + + it("honors saved early delivery off on initial render", () => { + setVoiceEarlyDeliveryPreference({ enabled: false }); + renderWithProviders(); + expect( + screen.getByRole("switch", { name: "Early delivery (experimental)" }), + ).not.toBeChecked(); + }); + beforeEach(async () => { + window.localStorage.clear(); await i18n.changeLanguage("en"); microphonePermissionState.status = "authorized"; microphonePermissionState.openSettingsError = false; diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index c67569fb9..a42d2a611 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -7,6 +7,8 @@ import { Alert, AlertDescription, AlertTitle } from "@/shared/ui/alert"; import { Button } from "@/shared/ui/button"; import { Badge } from "@/shared/ui/badge"; import { ConfirmDialog } from "@/shared/ui/confirm-dialog"; +import { Switch } from "@/shared/ui/switch"; +import { useVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { SettingsRow } from "@/shared/ui/settings-row"; import { Select, @@ -161,6 +163,7 @@ export function VoiceSettings() { setOpenAiVoice(openAiStatus.speechVoice); } }, [openAiStatus]); + const earlyDelivery = useVoiceEarlyDeliveryPreference(); const interruption = useVoiceInterruptionPreference(); const statusSounds = useStatusSoundPreference(); const siriSetup = useSiriVoiceSetup(output.backend === "siri"); @@ -234,6 +237,7 @@ export function VoiceSettings() { output.setBackend(getDefaultVoiceOutputBackend()); interruption.setMode(getDefaultVoiceInterruptionPreference().mode); statusSounds.update(getDefaultStatusSoundPreference()); + earlyDelivery.setEnabled(true); setResetDialogOpen(false); } finally { setResetting(false); @@ -591,6 +595,22 @@ export function VoiceSettings() { )} /> +
+ ( + + )} + /> +
{resetError ? (

{resetError} diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 6d532189a..a5af2e1d2 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1075,6 +1075,8 @@ "useVoice": "Use {{voice}}", "voice": "Voice", "voiceLabel": "Pocket TTS voice", + "earlyDelivery": "Early delivery (experimental)", + "earlyDeliveryDescription": "Prepare a private draft while you speak to reduce response time. Your final transcript remains authoritative.", "statusSounds": "Status sounds", "statusSoundModes": { "off": "Off", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index f1c10728a..7ba042cd8 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1074,6 +1074,8 @@ "useVoice": "Usar {{voice}}", "voice": "Voz", "voiceLabel": "Voz de Pocket TTS", + "earlyDelivery": "Entrega anticipada (experimental)", + "earlyDeliveryDescription": "Prepara un borrador privado mientras hablas para reducir el tiempo de respuesta. La transcripción final sigue siendo la definitiva.", "statusSounds": "Sonidos de estado", "statusSoundModes": { "off": "Desactivados", diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index f5e193a11..4f3abd684 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -261,6 +261,124 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { }); }); + if (scenario === "complete") + base( + "early delivery toggle persists and gates only private preparation", + async ({ page }) => { + const openSettings = async () => { + await page.locator("[data-sidebar-nav-id=settings]").click(); + await page.locator("[data-sidebar-nav-id=settings-voice]").click(); + }; + const openChat = async () => { + await page + .getByTestId("sidebar-primary-nav-panel") + .getByRole("button", { name: "Back", exact: true }) + .click(); + await page + .getByRole("button", { name: "Early voice", exact: true }) + .click(); + await expect(page.getByRole("textbox").first()).toBeVisible(); + }; + const toggle = page.getByRole("switch", { + name: "Early delivery (experimental)", + exact: true, + }); + const allRequests = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.acpRequests(), + ); + const forks = async () => + (await allRequests()).filter((r) => r.method === "session/fork"); + + await openSettings(); + await expect(toggle).toBeChecked(); + await toggle.uncheck(); + await expect(toggle).not.toBeChecked(); + await openChat(); + await page.goto("/"); + await page.reload(); + await openSettings(); + await expect(toggle).not.toBeChecked(); + await openChat(); + await nativeEvent(page, { + type: "startup", + ownerWindowLabel: "main", + line: "Ready", + }); + await nativeEvent(page, { type: "interimUser", ...utterance }); + // Drain renderer work before asserting that the disabled interim was ignored. + await page.evaluate( + () => + new Promise((resolve) => + requestAnimationFrame(() => + requestAnimationFrame(() => resolve()), + ), + ), + ); + expect(await forks()).toHaveLength(0); + expect(await requests(page)).toHaveLength(0); + await nativeEvent(page, { + type: "user", + ...utterance, + deliveryAttempts: 0, + }); + await expect.poll(() => requests(page)).toHaveLength(1); + const final = (await requests(page))[0]; + expect(final.method).toBe("session/prompt"); + expect(final.params?.sessionId).toBe(SESSION_ID); + expect(final.params?.prompt).toContainEqual({ + type: "text", + text: utterance.text, + }); + expect(await forks()).toHaveLength(0); + await expect( + page.getByText(utterance.text, { exact: true }), + ).toHaveCount(1); + await completePrompts(page); + + await openSettings(); + await toggle.check(); + await expect(toggle).toBeChecked(); + await openChat(); + await nativeEvent(page, { + type: "interimUser", + ...utterance, + id: "utterance-reenabled", + }); + await expect.poll(forks).toHaveLength(1); + await expect + .poll(async () => + (await allRequests()).filter( + (r) => r.method === "session/set_mode", + ), + ) + .toHaveLength(1); + const mode = (await allRequests()).find( + (r) => r.method === "session/set_mode", + ); + expect(mode?.params?.sessionId).not.toBe(SESSION_ID); + expect(mode?.params?.modeId).toBe("chat"); + await expect + .poll(async () => + (await requests(page)).filter( + (r) => r.params?.sessionId === mode?.params?.sessionId, + ), + ) + .toHaveLength(1); + expect( + (await requests(page)).filter( + (r) => r.params?.sessionId === SESSION_ID, + ), + ).toHaveLength(1); + await nativeEvent(page, { + type: "recognitionPending", + pending: false, + }); + }, + ); + for (const busy of [false, true]) base( `final has sole tool authority, bounded delivery and later utterance (busy: ${busy})`, From b51cf2c951b3c5122c053325c8d1595e8e1087f7 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:07:37 -0400 Subject: [PATCH 12/19] fix(voice): correlate cumulative Apple recognition segments Signed-off-by: John Matthew Tennant --- src-tauri/crates/berd-call/src/input.rs | 66 +++++++++++++++++++++---- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 771e91da7..51e904f6d 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -732,6 +732,9 @@ struct PendingRecognitions { /// The interim text last shown, so repeats and clears are sent once. interim: String, interim_recognition_id: String, + /// Apple recognition segments can span multiple VAD admission holds. + #[cfg(any(target_os = "macos", test))] + segment_id: Option, } impl PendingRecognitions { @@ -803,9 +806,36 @@ impl PendingRecognitions { self.unfinalized = false; } - /// Interim text is best effort: a full event queue drops it rather than stall recognition. + /// Apple reports cumulative text until a final, independent of VAD pauses. + #[cfg(any(target_os = "macos", test))] + fn show_segment_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { + let id = self + .segment_id + .get_or_insert_with(|| uuid::Uuid::new_v4().to_string()) + .clone(); + self.emit_interim(events, text, id); + } + + #[cfg(any(target_os = "macos", test))] + fn take_segment_recognition_id(&mut self) -> String { + self.segment_id + .take() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()) + } + + /// Turn-scoped backends retain the oldest final's identity after hold expiry. fn show_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { - let recognition_id = self.recognition_ids.back().cloned().unwrap_or_default(); + let id = self.recognition_ids.back().cloned().unwrap_or_default(); + self.emit_interim(events, text, id); + } + + /// Interim text is best effort: a full event queue drops it rather than stall recognition. + fn emit_interim( + &mut self, + events: &tokio_mpsc::Sender, + text: &str, + recognition_id: String, + ) { if self.interim != text || (!text.is_empty() && self.interim_recognition_id != recognition_id) { @@ -1552,13 +1582,9 @@ fn forward_mac_events( match event { crate::mac_speech::MacSpeechRecognitionEvent::Final(text, end) => { let text = text.trim().to_string(); + let recognition_id = pending.take_segment_recognition_id(); if !text.is_empty() { - send_final( - events, - pending.take_final_recognition_id(), - text, - storage_deadline, - )?; + send_final(events, recognition_id, text, storage_deadline)?; } // One final can span several turns split by pauses, and one turn // can produce several finals, so a final with an end resolves @@ -1577,7 +1603,7 @@ fn forward_mac_events( } crate::mac_speech::MacSpeechRecognitionEvent::Heard(_, text) => { pending.unfinalized = true; - pending.show_interim(events, &text); + pending.show_segment_interim(events, &text); } crate::mac_speech::MacSpeechRecognitionEvent::Finished => { if storage_deadline.is_none() { @@ -3146,6 +3172,28 @@ mod tests { ); } + #[test] + fn apple_cumulative_segment_survives_pause_turns() { + let (events, mut receiver) = tokio_mpsc::channel(16); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.show_segment_interim(&events, "first words"); + pending.end_turn(&events, 100).unwrap(); + pending.begin(&events).unwrap(); + pending.show_segment_interim(&events, "first words and more"); + let mut ids = Vec::new(); + while let Ok(event) = receiver.try_recv() { + if let VoiceInputEvent::InterimTranscript { recognition_id, .. } = event { + ids.push(recognition_id); + } + } + assert_eq!(ids[0], ids[1], "one Apple segment across two VAD turns"); + pending.release_hold(&events).unwrap(); + assert_eq!(pending.take_segment_recognition_id(), ids[1]); + pending.show_segment_interim(&events, "next segment"); + assert_ne!(pending.take_segment_recognition_id(), ids[1]); + } + #[test] fn expired_recognition_keeps_identity_after_new_provisional() { let (events, mut receiver) = tokio_mpsc::channel(16); From af5c6ebf1a1afb564395f5cb32eeaf6f781139fe Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:07:37 -0400 Subject: [PATCH 13/19] fix(voice): use parent experiment configuration for early delivery Signed-off-by: John Matthew Tennant --- .../experiments/experimentDefinitions.ts | 8 + .../lib/voiceEarlyDeliveryPreference.test.ts | 85 ++++++--- .../lib/voiceEarlyDeliveryPreference.ts | 170 +++++------------- .../voice-conversation/ui/VoiceSettings.tsx | 11 +- src/shared/i18n/locales/en/settings.json | 1 + src/shared/i18n/locales/es/settings.json | 1 + 6 files changed, 130 insertions(+), 146 deletions(-) diff --git a/src/features/experiments/experimentDefinitions.ts b/src/features/experiments/experimentDefinitions.ts index 54bccb0bb..1916a1386 100644 --- a/src/features/experiments/experimentDefinitions.ts +++ b/src/features/experiments/experimentDefinitions.ts @@ -100,6 +100,14 @@ export const EXPERIMENT_DEFINITIONS = [ titleKey: "experiments.voiceConversation.title", descriptionKey: "experiments.voiceConversation.description", defaultEnabled: true, + config: { + earlyDelivery: { + type: "boolean", + labelKey: "voice.earlyDelivery", + descriptionKey: "voice.earlyDeliveryDescription", + defaultValue: true, + }, + }, }, { id: CHAT_ON_CANVAS_EXPERIMENT_ID, diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts index baa18e3a4..d3c8118f2 100644 --- a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts @@ -1,40 +1,83 @@ import { act, renderHook } from "@testing-library/react"; -import { beforeEach, describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { VOICE_CONVERSATION_EXPERIMENT_ID as id } from "@/features/experiments/experimentDefinitions"; +import { + EXPERIMENT_PREFERENCES_STORAGE_KEY as key, + getExperiment, + setExperimentConfigValue, + setExperimentEnabled, +} from "@/features/experiments/experimentPreferences"; import { getVoiceEarlyDeliveryPreference, setVoiceEarlyDeliveryPreference, useVoiceEarlyDeliveryPreference, } from "./voiceEarlyDeliveryPreference"; -const key = "goose:voice-early-delivery-preference"; - -describe("voice early delivery preference", () => { +describe("voice early delivery experiment config", () => { beforeEach(() => window.localStorage.clear()); - - it("defaults on and preserves saved off", () => { - expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: true }); + it("defaults on inside enabled voice experiment", () => { + setExperimentEnabled(id, true); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + setExperimentEnabled(id, false); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); + }); + it("writes config under parent and preserves saved off", () => { setVoiceEarlyDeliveryPreference({ enabled: false }); - expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: false }); - expect(JSON.parse(window.localStorage.getItem(key) ?? "null")).toEqual({ - enabled: false, - }); + expect(getExperiment(id)?.config.earlyDelivery).toBe(false); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); }); - - it.each([ - "invalid json", - "{}", - '{"enabled":"false"}', - ])("defaults on for invalid storage: %s", (raw) => { - window.localStorage.setItem(key, raw); - expect(getVoiceEarlyDeliveryPreference()).toEqual({ enabled: true }); + it("migrates explicitly saved legacy off without overwriting registry choices", () => { + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); + expect(getExperiment(id)?.config.earlyDelivery).toBe(false); + setExperimentConfigValue(id, "earlyDelivery", true); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + }); + it("preserves an existing registry choice over legacy off", () => { + setExperimentConfigValue(id, "earlyDelivery", true); + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + }); + it("reports failed and unsupported-version writes without losing saved off", () => { + window.localStorage.setItem(key, '{"version":999}'); + expect(setVoiceEarlyDeliveryPreference({ enabled: false })).toBe(false); + expect(window.localStorage.getItem(key)).toBe('{"version":999}'); + window.localStorage.clear(); + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + const write = vi + .spyOn(Storage.prototype, "setItem") + .mockImplementation(() => { + throw new Error("denied"); + }); + expect(setVoiceEarlyDeliveryPreference({ enabled: false })).toBe(false); + expect( + window.localStorage.getItem("goose:voice-early-delivery-preference"), + ).toBe('{"enabled":false}'); + write.mockRestore(); }); - it("updates same-window and cross-window subscribers", () => { const { result } = renderHook(() => useVoiceEarlyDeliveryPreference()); act(() => result.current.setEnabled(false)); expect(result.current.enabled).toBe(false); act(() => { - window.localStorage.setItem(key, JSON.stringify({ enabled: true })); + window.localStorage.setItem( + key, + JSON.stringify({ + version: 2, + experiments: { + [id]: { enabled: true, config: { earlyDelivery: true } }, + }, + }), + ); window.dispatchEvent(new StorageEvent("storage", { key })); }); expect(result.current.enabled).toBe(true); diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts index 1f6af5d09..58d262fc9 100644 --- a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts @@ -1,146 +1,70 @@ -import { useCallback, useSyncExternalStore } from "react"; +import { useCallback } from "react"; +import { VOICE_CONVERSATION_EXPERIMENT_ID as id } from "@/features/experiments/experimentDefinitions"; +import { + EXPERIMENT_PREFERENCES_STORAGE_KEY, + getExperiment, + setExperimentConfigValue, + subscribeToExperimentChanges, + useExperiment, +} from "@/features/experiments/experimentPreferences"; export interface VoiceEarlyDeliveryPreference { enabled: boolean; } -const STORAGE_KEY = "goose:voice-early-delivery-preference"; -const CHANGED_EVENT = "goose:voice-early-delivery-preference-changed"; -const DEFAULT_PREFERENCE: VoiceEarlyDeliveryPreference = { - enabled: true, -}; -const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); -let volatilePreference: - | { - preference: VoiceEarlyDeliveryPreference; - storageValueBeforeWrite: string | null | undefined; - } - | undefined; -let removeVolatileStorageListener: (() => void) | undefined; - -function normalize(value: unknown): VoiceEarlyDeliveryPreference { - if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; - const candidate = value as Partial; - return { - enabled: - typeof candidate.enabled === "boolean" - ? candidate.enabled - : DEFAULT_PREFERENCE.enabled, - }; -} - -export function getDefaultVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { - return DEFAULT_PREFERENCE; -} - -export function getVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { - if (typeof window === "undefined") return DEFAULT_PREFERENCE; +// Compatibility for the briefly published standalone switch. Registry choices win. +function migrateLegacyOff() { + if (typeof window === "undefined") return; + const legacyKey = "goose:voice-early-delivery-preference"; try { - const raw = window.localStorage.getItem(STORAGE_KEY); - if (volatilePreference) { - if ( - volatilePreference.storageValueBeforeWrite === undefined || - raw === volatilePreference.storageValueBeforeWrite - ) { - return volatilePreference.preference; - } - clearVolatilePreference(); - } - return raw ? normalize(JSON.parse(raw)) : DEFAULT_PREFERENCE; + const legacy = window.localStorage.getItem(legacyKey); + if (!legacy) return; + const stored = JSON.parse( + window.localStorage.getItem(EXPERIMENT_PREFERENCES_STORAGE_KEY) ?? "{}", + ); + if ( + stored.experiments?.[id]?.config?.earlyDelivery === undefined && + JSON.parse(legacy)?.enabled === false && + !setExperimentConfigValue(id, "earlyDelivery", false) + ) + return; + window.localStorage.removeItem(legacyKey); } catch { - return volatilePreference?.preference ?? DEFAULT_PREFERENCE; + // Leave failed migrations intact for the next read. } } -function getSnapshot(): string { - return JSON.stringify(getVoiceEarlyDeliveryPreference()); -} - -const listeners = new Set<() => void>(); -let removeWindowListeners: (() => void) | undefined; - -function notify() { - for (const listener of listeners) listener(); -} - -function clearVolatilePreference() { - volatilePreference = undefined; - removeVolatileStorageListener?.(); - removeVolatileStorageListener = undefined; +export function getDefaultVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + return { enabled: true }; } -function retainVolatilePreference( - preference: VoiceEarlyDeliveryPreference, - storageValueBeforeWrite: string | null | undefined, -) { - volatilePreference = { preference, storageValueBeforeWrite }; - if (removeVolatileStorageListener || typeof window === "undefined") return; - const handleStorage = (event: StorageEvent) => { - if (event.key !== STORAGE_KEY && event.key !== null) return; - clearVolatilePreference(); - notify(); - }; - window.addEventListener("storage", handleStorage); - removeVolatileStorageListener = () => { - window.removeEventListener("storage", handleStorage); +export function getVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + migrateLegacyOff(); + const experiment = getExperiment(id); + return { + enabled: !!experiment?.enabled && experiment.config.earlyDelivery === true, }; } -export function subscribeToVoiceEarlyDeliveryPreference(listener: () => void) { - if (typeof window === "undefined") return () => {}; - listeners.add(listener); - if (!removeWindowListeners) { - const handleStorage = (event: StorageEvent) => { - if (event.key === STORAGE_KEY || event.key === null) { - clearVolatilePreference(); - notify(); - } - }; - window.addEventListener(CHANGED_EVENT, notify); - window.addEventListener("storage", handleStorage); - removeWindowListeners = () => { - window.removeEventListener(CHANGED_EVENT, notify); - window.removeEventListener("storage", handleStorage); - }; - } - return () => { - listeners.delete(listener); - if (listeners.size === 0) { - removeWindowListeners?.(); - removeWindowListeners = undefined; - } - }; -} +export const subscribeToVoiceEarlyDeliveryPreference = + subscribeToExperimentChanges; export function setVoiceEarlyDeliveryPreference( preference: VoiceEarlyDeliveryPreference, -): void { - if (typeof window === "undefined") return; - const value = normalize(preference); - let storageValueBeforeWrite: string | null | undefined; - try { - storageValueBeforeWrite = window.localStorage.getItem(STORAGE_KEY); - window.localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); - clearVolatilePreference(); - } catch { - // Keep the current in-memory renderer usable when storage is unavailable. - retainVolatilePreference(value, storageValueBeforeWrite); - } - window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: value })); +): boolean { + migrateLegacyOff(); + return setExperimentConfigValue(id, "earlyDelivery", preference.enabled); } export function useVoiceEarlyDeliveryPreference() { - const snapshot = useSyncExternalStore( - subscribeToVoiceEarlyDeliveryPreference, - getSnapshot, - () => DEFAULT_SNAPSHOT, + migrateLegacyOff(); + const experiment = useExperiment(id); + const setEnabled = useCallback( + (enabled: boolean) => setVoiceEarlyDeliveryPreference({ enabled }), + [], ); - const preference = normalize(JSON.parse(snapshot)); - const setEnabled = useCallback((enabled: boolean) => { - setVoiceEarlyDeliveryPreference({ - ...getVoiceEarlyDeliveryPreference(), - enabled, - }); - }, []); - return { ...preference, setEnabled }; + return { + enabled: !!experiment?.enabled && experiment.config.earlyDelivery === true, + setEnabled, + }; } diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index a42d2a611..14592a292 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -237,7 +237,8 @@ export function VoiceSettings() { output.setBackend(getDefaultVoiceOutputBackend()); interruption.setMode(getDefaultVoiceInterruptionPreference().mode); statusSounds.update(getDefaultStatusSoundPreference()); - earlyDelivery.setEnabled(true); + if (!earlyDelivery.setEnabled(true)) + throw new Error(t("voice.earlyDeliverySaveFailed")); setResetDialogOpen(false); } finally { setResetting(false); @@ -606,7 +607,13 @@ export function VoiceSettings() { aria-labelledby={labelId} aria-describedby={descriptionId} checked={earlyDelivery.enabled} - onCheckedChange={earlyDelivery.setEnabled} + onCheckedChange={(enabled) => { + setResetError( + earlyDelivery.setEnabled(enabled) + ? null + : t("voice.earlyDeliverySaveFailed"), + ); + }} /> )} /> diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index a5af2e1d2..ca4cc32fd 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1075,6 +1075,7 @@ "useVoice": "Use {{voice}}", "voice": "Voice", "voiceLabel": "Pocket TTS voice", + "earlyDeliverySaveFailed": "Could not save early delivery setting.", "earlyDelivery": "Early delivery (experimental)", "earlyDeliveryDescription": "Prepare a private draft while you speak to reduce response time. Your final transcript remains authoritative.", "statusSounds": "Status sounds", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 7ba042cd8..fbed65f6e 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1074,6 +1074,7 @@ "useVoice": "Usar {{voice}}", "voice": "Voz", "voiceLabel": "Voz de Pocket TTS", + "earlyDeliverySaveFailed": "No se pudo guardar la configuración de entrega anticipada.", "earlyDelivery": "Entrega anticipada (experimental)", "earlyDeliveryDescription": "Prepara un borrador privado mientras hablas para reducir el tiempo de respuesta. La transcripción final sigue siendo la definitiva.", "statusSounds": "Sonidos de estado", From 5b7e9b5146d6d3d64a4fb927e4390655876212dc Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:07:38 -0400 Subject: [PATCH 14/19] fix(voice): create draft sessions hidden from the outset Signed-off-by: John Matthew Tennant --- .../lib/privateVoiceDraft.test.ts | 49 ++++++++++-- .../lib/privateVoiceDraft.ts | 38 ++++++--- tests/e2e/fixtures/tauri-mock.ts | 18 +++-- tests/e2e/voice-conversation.spec.ts | 77 ++++++++++++++++--- 4 files changed, 148 insertions(+), 34 deletions(-) diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts index 982682655..e3acb86e3 100644 --- a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts +++ b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts @@ -2,6 +2,9 @@ import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { startPrivateVoiceDraft } from "./privateVoiceDraft"; const mocks = vi.hoisted(() => ({ forkSession: vi.fn(), + newSession: vi.fn(), + getSessionInfo: vi.fn(), + setModel: vi.fn(), setSessionMode: vi.fn(), prompt: vi.fn(), cancelSession: vi.fn(), @@ -24,7 +27,12 @@ function deferred() { beforeEach(() => { vi.useFakeTimers(); vi.resetAllMocks(); - mocks.forkSession.mockResolvedValue({ sessionId: "private" }); + mocks.getSessionInfo.mockResolvedValue({ + providerId: "openai", + modelId: "configured-model", + }); + mocks.newSession.mockResolvedValue({ sessionId: "private" }); + mocks.setModel.mockResolvedValue(undefined); mocks.setSessionMode.mockResolvedValue(undefined); mocks.prompt.mockResolvedValue(undefined); mocks.cancelSession.mockResolvedValue(undefined); @@ -51,7 +59,17 @@ it("awaits chat mode before first prompt and captures only private text", async mode.resolve(); await vi.advanceTimersByTimeAsync(0); expect(await draft.take()).toBe("Draft"); - expect(mocks.forkSession).toHaveBeenCalledWith("main", "/work"); + expect(mocks.forkSession).not.toHaveBeenCalled(); + expect(mocks.getSessionInfo).toHaveBeenCalledWith("main"); + expect(mocks.newSession).toHaveBeenCalledWith("/work", { + hidden: true, + providerId: "openai", + backendId: "local", + }); + expect(mocks.setModel).toHaveBeenCalledWith("private", "configured-model"); + expect(mocks.prompt).toHaveBeenCalledWith("private", [ + { type: "text", text: "interim" }, + ]); expect(mocks.cancelSession).toHaveBeenCalledWith("private"); }); it("fails closed when chat mode is rejected", async () => { @@ -61,14 +79,14 @@ it("fails closed when chat mode is rejected", async () => { expect(mocks.prompt).not.toHaveBeenCalled(); }); it.each([ - "fork", + "creation", "mode", "prompt", ])("bounds stalled %s and cancellation without touching main", async (stage) => { const stalled = deferred(); mocks[ - stage === "fork" - ? "forkSession" + stage === "creation" + ? "newSession" : stage === "mode" ? "setSessionMode" : "prompt" @@ -86,7 +104,7 @@ it.each([ }); it("bounds best effort setup and cleans a fork created after disposal", async () => { const fork = deferred<{ sessionId: string }>(); - mocks.forkSession.mockReturnValue(fork.promise); + mocks.newSession.mockReturnValue(fork.promise); startPrivateVoiceDraft("main", "/work", "interim"); await vi.advanceTimersByTimeAsync(2000); fork.resolve({ sessionId: "late-private" }); @@ -105,3 +123,22 @@ it("never prompts after a timed out mode ack", async () => { await vi.advanceTimersByTimeAsync(0); expect(mocks.prompt).not.toHaveBeenCalled(); }); + +it.each([ + { providerId: null, modelId: "model" }, + { providerId: "openai", modelId: null }, +])("fails closed without configured provider/model: %o", async (config) => { + mocks.getSessionInfo.mockResolvedValue(config); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await expect(draft.take()).resolves.toBeNull(); + expect(mocks.newSession).not.toHaveBeenCalled(); + expect(mocks.forkSession).not.toHaveBeenCalled(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("fails closed on model configuration failure", async () => { + mocks.setModel.mockRejectedValue(new Error("unavailable")); + await expect( + startPrivateVoiceDraft("main", "/work", "interim").take(), + ).resolves.toBeNull(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.ts b/src/features/voice-conversation/lib/privateVoiceDraft.ts index c11a48d4a..ec90475e6 100644 --- a/src/features/voice-conversation/lib/privateVoiceDraft.ts +++ b/src/features/voice-conversation/lib/privateVoiceDraft.ts @@ -1,7 +1,9 @@ import { cancelSession, deleteSession, - forkSession, + getSessionInfo, + newSession, + setModel, prompt, setSessionMode, } from "@/shared/api/acpApi"; @@ -10,6 +12,8 @@ import { interceptSessionNotifications, } from "@/shared/api/acpConnection"; +import { getSessionBackend } from "@/shared/api/acpSessionBackends"; + export interface PrivateVoiceDraft { take: () => Promise; dispose: () => void; @@ -22,10 +26,10 @@ export function startPrivateVoiceDraft( text: string, ): PrivateVoiceDraft { let stopped = false; - let forkId: string | undefined; + let privateSessionId: string | undefined; let stopIntercepting: (() => void) | undefined; let setupTimer: ReturnType | undefined; - const cleanupFork = (sessionId: string) => { + const cleanupSession = (sessionId: string) => { hidePrivateSessionNotifications(sessionId); // Neither cancellation nor deletion is trusted to settle. They cannot hold // the final queue, and neither ever targets the source session. @@ -38,26 +42,38 @@ export function startPrivateVoiceDraft( clearTimeout(setupTimer); clearTimeout(lifetimeTimer); stopIntercepting?.(); - if (forkId) cleanupFork(forkId); + if (privateSessionId) cleanupSession(privateSessionId); }; const lifetimeTimer = setTimeout(dispose, 15_000); setupTimer = setTimeout(dispose, 2_000); const result = (async () => { - const fork = await forkSession(sourceSessionId, workingDir); - forkId = fork.sessionId; - hidePrivateSessionNotifications(forkId); + const source = await getSessionInfo(sourceSessionId); + if (stopped || !source.providerId?.trim() || !source.modelId?.trim()) + return null; + // Transcript-only speculation: never copy source history (including hidden + // context). The final source-session turn retains full history and authority. + // Hidden must be set at creation, not applied to a visible fork afterward. + const session = await newSession(workingDir, { + hidden: true, + providerId: source.providerId, + backendId: getSessionBackend(sourceSessionId), + }); + privateSessionId = session.sessionId; + hidePrivateSessionNotifications(privateSessionId); if (stopped) { - cleanupFork(forkId); + cleanupSession(privateSessionId); return null; } // Fail closed. Backend chat mode skips ALL tool_requests, including tools // that bypass the permission callback. Prompt must never precede this ack. - await setSessionMode(forkId, "chat"); + await setModel(privateSessionId, source.modelId); + if (stopped) return null; + await setSessionMode(privateSessionId, "chat"); if (stopped) return null; clearTimeout(setupTimer); const chunks: string[] = []; stopIntercepting = interceptSessionNotifications((notification) => { - if (notification.sessionId !== forkId) return false; + if (notification.sessionId !== privateSessionId) return false; const { update } = notification; if ( !stopped && @@ -67,7 +83,7 @@ export function startPrivateVoiceDraft( chunks.push(update.content.text); return true; }); - await prompt(forkId, [{ type: "text", text }]); + await prompt(privateSessionId, [{ type: "text", text }]); return stopped ? null : chunks.join("").trim() || null; })() .catch(() => null) diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index b97e58336..fa5787f20 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -218,7 +218,7 @@ export function buildInitScript(options?: { let nextEventId = 1; localStorage.setItem("goose:defaultProvider", "goose"); - if (ENABLED_EXPERIMENTS.length > 0) { + if (ENABLED_EXPERIMENTS.length > 0 && !localStorage.getItem("goose:experimental-features")) { localStorage.setItem( "goose:experimental-features", JSON.stringify({ @@ -426,7 +426,7 @@ export function buildInitScript(options?: { }); case "session/list": return jsonRpcResult(message.id, { - sessions: ACP_SESSIONS.map((session) => ({ + sessions: ACP_SESSIONS.filter((session) => !session.hidden).map((session) => ({ sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, @@ -440,10 +440,14 @@ export function buildInitScript(options?: { cwd: session.workingDir, })), }); + case "_goose/unstable/session/info": { + const session = findSession(message.params?.sessionId); + return jsonRpcResult(message.id, { session: { sessionId: session.sessionId, cwd: session.cwd, _meta: { providerId: session.providerId, modelId: session.modelId } } }); + } case "session/new": { - const providerId = message.params?.meta?.provider ?? "goose"; + const providerId = message.params?._meta?.provider ?? "goose"; const sessionId = "session-" + Math.random().toString(36).slice(2, 10); - ACP_SESSIONS.unshift(buildSession(sessionId, providerId)); + ACP_SESSIONS.unshift({ ...buildSession(sessionId, providerId), hidden: message.params?._meta?.hidden === true, cwd: message.params?.cwd }); return jsonRpcResult(message.id, { sessionId }); } case "session/fork": { @@ -726,9 +730,9 @@ export function buildInitScript(options?: { ACP_REQUESTS.push(clone(message)); // Semantic model boundary: private inference is only accepted after // a successful chat-mode transition, not merely an audience hint. - if (PRIVATE_VOICE_DRAFT && message.method === "session/prompt" && message.params.sessionId.startsWith("session-fork-")) { + if (PRIVATE_VOICE_DRAFT && message.method === "session/prompt" && findSession(message.params.sessionId)?.hidden) { const session = findSession(message.params.sessionId); - if (session?.mode !== "chat") throw new Error("Private model invoked with tool authority"); + if (session?.mode !== "chat" || !session.providerId || !session.modelId) throw new Error("Private model invoked without chat mode and configured model"); if (PRIVATE_VOICE_DRAFT === "complete") { queueMicrotask(() => { window.__GOOSE_E2E__.emitAcpNotification({ sessionId: session.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "PRIVATE DRAFT: I would set the temperature to 10 degrees." } } }); @@ -987,7 +991,7 @@ export function buildInitScript(options?: { return Promise.resolve(null); case "list_sessions": return Promise.resolve( - ACP_SESSIONS.map((session) => ({ + ACP_SESSIONS.filter((session) => !session.hidden).map((session) => ({ sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 4f3abd684..c8dce6716 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -289,8 +289,11 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { window as typeof window & { __GOOSE_E2E__: VoiceHarness } ).__GOOSE_E2E__.acpRequests(), ); - const forks = async () => - (await allRequests()).filter((r) => r.method === "session/fork"); + const hiddenSessions = async () => + (await allRequests()).filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); await openSettings(); await expect(toggle).toBeChecked(); @@ -317,7 +320,7 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { ), ), ); - expect(await forks()).toHaveLength(0); + expect(await hiddenSessions()).toHaveLength(0); expect(await requests(page)).toHaveLength(0); await nativeEvent(page, { type: "user", @@ -332,7 +335,7 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { type: "text", text: utterance.text, }); - expect(await forks()).toHaveLength(0); + expect(await hiddenSessions()).toHaveLength(0); await expect( page.getByText(utterance.text, { exact: true }), ).toHaveCount(1); @@ -347,7 +350,7 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { ...utterance, id: "utterance-reenabled", }); - await expect.poll(forks).toHaveLength(1); + await expect.poll(hiddenSessions).toHaveLength(1); await expect .poll(async () => (await allRequests()).filter( @@ -360,6 +363,31 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { ); expect(mode?.params?.sessionId).not.toBe(SESSION_ID); expect(mode?.params?.modeId).toBe("chat"); + const creationWire = await allRequests(); + expect( + creationWire.filter((r) => r.method === "session/fork"), + ).toHaveLength(0); + const created = creationWire.filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); + expect(created).toHaveLength(1); + expect(created[0].params?.cwd).toBe("/workspace"); + expect(created[0].params?._meta).toMatchObject({ + hidden: true, + provider: "openai", + }); + expect(creationWire).toContainEqual( + expect.objectContaining({ + method: "session/set_config_option", + params: expect.objectContaining({ + sessionId: mode?.params?.sessionId, + configId: "model", + value: "gpt-4.1", + }), + }), + ); + await expect .poll(async () => (await requests(page)).filter( @@ -407,9 +435,34 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { (r) => r.method === "session/set_mode", ); const privateId = mode?.params?.sessionId; - if (!privateId) throw new Error("Missing private fork"); + if (!privateId) throw new Error("Missing hidden private session"); expect(privateId).not.toBe(SESSION_ID); expect(mode?.params?.modeId).toBe("chat"); + const creationWire = await allRequests(); + expect( + creationWire.filter((r) => r.method === "session/fork"), + ).toHaveLength(0); + const created = creationWire.filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); + expect(created).toHaveLength(1); + expect(created[0].params?.cwd).toBe("/workspace"); + expect(created[0].params?._meta).toMatchObject({ + hidden: true, + provider: "openai", + }); + expect(creationWire).toContainEqual( + expect.objectContaining({ + method: "session/set_config_option", + params: expect.objectContaining({ + sessionId: mode?.params?.sessionId, + configId: "model", + value: "gpt-4.1", + }), + }), + ); + if (scenario !== "mode-rejected") { await expect.poll(() => requests(page)).toHaveLength(1); expect((await requests(page))[0].params?.sessionId).toBe(privateId); @@ -525,22 +578,26 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { "recognition ending without final permits another private utterance", async ({ page }) => { await nativeEvent(page, { type: "interimUser", ...utterance }); - const forks = () => + const hiddenSessions = () => page.evaluate(() => ( window as typeof window & { __GOOSE_E2E__: VoiceHarness } ).__GOOSE_E2E__ .acpRequests() - .filter((r) => r.method === "session/fork"), + .filter( + (r) => + r.method === "session/new" && + r.params?._meta?.hidden === true, + ), ); - await expect.poll(forks).toHaveLength(1); + await expect.poll(hiddenSessions).toHaveLength(1); await nativeEvent(page, { type: "recognitionPending", pending: false }); await nativeEvent(page, { type: "interimUser", ...utterance, id: "utterance-2", }); - await expect.poll(forks).toHaveLength(2); + await expect.poll(hiddenSessions).toHaveLength(2); expect( (await requests(page)).every( (r) => r.params?.sessionId !== SESSION_ID, From c866a5538698f4ac8ec93ca7a82607f6b7126254 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:13:59 -0400 Subject: [PATCH 15/19] test(voice): describe delayed private session creation Signed-off-by: John Matthew Tennant --- .../voice-conversation/lib/privateVoiceDraft.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts index e3acb86e3..f1ecbb7e9 100644 --- a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts +++ b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts @@ -102,12 +102,12 @@ it.each([ ); if (stage !== "prompt") expect(mocks.prompt).not.toHaveBeenCalled(); }); -it("bounds best effort setup and cleans a fork created after disposal", async () => { - const fork = deferred<{ sessionId: string }>(); - mocks.newSession.mockReturnValue(fork.promise); +it("bounds best effort setup and cleans a private session created after disposal", async () => { + const creation = deferred<{ sessionId: string }>(); + mocks.newSession.mockReturnValue(creation.promise); startPrivateVoiceDraft("main", "/work", "interim"); await vi.advanceTimersByTimeAsync(2000); - fork.resolve({ sessionId: "late-private" }); + creation.resolve({ sessionId: "late-private" }); await vi.advanceTimersByTimeAsync(0); expect(mocks.cancelSession).toHaveBeenCalledWith("late-private"); expect(mocks.hide).toHaveBeenCalledWith("late-private"); From 43dfcf91a6fc4280cbc50799f5b5b9e8d8d19112 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:21:55 -0400 Subject: [PATCH 16/19] refactor(voice): remove unused turn-scoped identity ledger Signed-off-by: John Matthew Tennant --- src-tauri/crates/berd-call/src/input.rs | 134 +++++++++--------------- 1 file changed, 49 insertions(+), 85 deletions(-) diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 51e904f6d..f8bdfee61 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -719,8 +719,6 @@ struct PendingRecognitions { ambiguous: bool, /// Where each unresolved turn's speech ended, oldest first; None while it continues. ends: VecDeque>, - /// Stable identities survive admission-hold expiry until recognition resolves. - recognition_ids: VecDeque, /// Where the furthest final so far ended, since one can arrive before its turn ends. #[cfg(any(target_os = "macos", test))] final_end: Option, @@ -756,8 +754,6 @@ impl PendingRecognitions { } else { self.count += 1; self.ends.push_back(None); - self.recognition_ids - .push_back(uuid::Uuid::new_v4().to_string()); } let was_blocking = self.blocking > 0; self.blocking = self.blocking.saturating_add(1).min(limit); @@ -823,10 +819,8 @@ impl PendingRecognitions { .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()) } - /// Turn-scoped backends retain the oldest final's identity after hold expiry. - fn show_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { - let id = self.recognition_ids.back().cloned().unwrap_or_default(); - self.emit_interim(events, text, id); + fn clear_interim(&mut self, events: &tokio_mpsc::Sender) { + self.emit_interim(events, "", String::new()); } /// Interim text is best effort: a full event queue drops it rather than stall recognition. @@ -848,15 +842,6 @@ impl PendingRecognitions { } } - fn take_final_recognition_id(&mut self) -> String { - // Apple can emit several finals before one VAD turn ends. Only the - // first final completes its provisional; later segments need fresh IDs. - let next = uuid::Uuid::new_v4().to_string(); - self.recognition_ids - .front_mut() - .map_or_else(|| next.clone(), |id| std::mem::replace(id, next.clone())) - } - fn result_is_current(&self) -> bool { !self.ambiguous && self.count == self.blocking } @@ -868,7 +853,6 @@ impl PendingRecognitions { let current = self.result_is_current(); self.count -= 1; self.ends.pop_front(); - self.recognition_ids.pop_front(); if current { self.blocking -= 1; if self.blocking == 0 { @@ -892,7 +876,7 @@ impl PendingRecognitions { fn reset(&mut self, events: &tokio_mpsc::Sender) -> Result<(), ()> { self.release_hold(events)?; - self.show_interim(events, ""); + self.clear_interim(events); *self = Self::new(); Ok(()) } @@ -1547,7 +1531,7 @@ fn complete_parakeet_recognition( if !text.is_empty() { let _ = send_final( events, - pending.take_final_recognition_id(), + uuid::Uuid::new_v4().to_string(), text, storage_deadline, ); @@ -1598,7 +1582,7 @@ fn forward_mac_events( *settle_deadline = None; } if !pending.words_outstanding() { - pending.show_interim(events, ""); + pending.clear_interim(events); } } crate::mac_speech::MacSpeechRecognitionEvent::Heard(_, text) => { @@ -1732,7 +1716,7 @@ fn mac_speech_worker( return; } pending.abandon_heard_words(); - pending.show_interim(&events, ""); + pending.clear_interim(&events); settle_deadline = None; } let frame = match frames.recv_timeout(Duration::from_millis(50)) { @@ -2011,7 +1995,7 @@ fn deliver_openai_turns( if !text.is_empty() { let _ = send_final( events, - pending.take_final_recognition_id(), + uuid::Uuid::new_v4().to_string(), text, storage_deadline, ); @@ -3195,76 +3179,56 @@ mod tests { } #[test] - fn expired_recognition_keeps_identity_after_new_provisional() { - let (events, mut receiver) = tokio_mpsc::channel(16); - let mut pending = PendingRecognitions::new(); - pending.begin(&events).unwrap(); - pending.show_interim(&events, "old provisional"); - pending.release_hold(&events).unwrap(); - pending.begin(&events).unwrap(); - pending.show_interim(&events, "new provisional"); - complete_parakeet_recognition("old final".into(), &events, &mut pending, None); - assert_eq!( - pending.blocking, 1, - "old final must not release the new hold" - ); - complete_parakeet_recognition("new final".into(), &events, &mut pending, None); - let mut provisionals = Vec::new(); - let mut finals = Vec::new(); - while let Ok(event) = receiver.try_recv() { - match event { - VoiceInputEvent::InterimTranscript { - recognition_id, - text, - } => { - provisionals.push((recognition_id, text)); + fn final_only_backends_generate_distinct_recognition_ids() { + for openai in [false, true] { + let (events, mut receiver) = tokio_mpsc::channel(16); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.release_hold(&events).unwrap(); + pending.begin(&events).unwrap(); + for text in ["old final", "new final"] { + if openai { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: text.into(), + mute_epoch: 0, + settle_deadline: Instant::now(), + }]); + let mut completed = HashMap::from([(text.into(), text.into())]); + deliver_openai_turns( + &mut committed, + &mut completed, + &events, + &mut pending, + None, + None, + ); + } else { + complete_parakeet_recognition(text.into(), &events, &mut pending, None); } - VoiceInputEvent::FinalTranscript { + assert_eq!(pending.blocking, usize::from(text == "old final")); + } + let mut finals = Vec::new(); + while let Ok(event) = receiver.try_recv() { + if let VoiceInputEvent::FinalTranscript { recognition_id, text, .. - } => { + } = event + { + assert_eq!( + uuid::Uuid::parse_str(&recognition_id) + .unwrap() + .get_version_num(), + 4 + ); finals.push((recognition_id, text)); } - _ => {} - } - } - assert_eq!(provisionals.len(), 2); - assert_eq!(finals.len(), 2); - assert_ne!(provisionals[0].0, provisionals[1].0); - assert_eq!(finals[0], (provisionals[0].0.clone(), "old final".into())); - assert_eq!(finals[1], (provisionals[1].0.clone(), "new final".into())); - assert_eq!(pending.blocking, 0); - assert!(pending.recognition_ids.is_empty()); - } - - #[test] - fn identical_interims_on_new_turns_keep_distinct_identities() { - let (events, mut receiver) = tokio_mpsc::channel(8); - let mut pending = PendingRecognitions::new(); - pending.begin(&events).unwrap(); - pending.show_interim(&events, "same words"); - pending.release_hold(&events).unwrap(); - pending.begin(&events).unwrap(); - pending.show_interim(&events, "same words"); - let mut ids = Vec::new(); - while let Ok(event) = receiver.try_recv() { - if let VoiceInputEvent::InterimTranscript { recognition_id, .. } = event { - ids.push(recognition_id); } + assert_eq!(finals.len(), 2); + assert_ne!(finals[0].0, finals[1].0); + assert_eq!(finals[0].1, "old final"); + assert_eq!(finals[1].1, "new final"); } - assert_eq!(ids.len(), 2); - assert_ne!(ids[0], ids[1]); - } - - #[test] - fn several_finals_before_turn_end_do_not_reuse_the_provisional_identity() { - let (events, _receiver) = tokio_mpsc::channel(2); - let mut pending = PendingRecognitions::new(); - pending.begin(&events).unwrap(); - let provisional = pending.recognition_ids.front().unwrap().clone(); - assert_eq!(pending.take_final_recognition_id(), provisional); - assert_ne!(pending.take_final_recognition_id(), provisional); } #[test] From a858efe4762915077d128d97d74268cbc8f7635b Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 18:32:50 -0400 Subject: [PATCH 17/19] test(voice): preserve session paths after monitor fixture integration Signed-off-by: John Matthew Tennant --- tests/e2e/fixtures/tauri-mock.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index fa5787f20..7965ece87 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -437,7 +437,6 @@ export function buildInitScript(options?: { providerId: session.providerId, modelId: session.modelId, }, - cwd: session.workingDir, })), }); case "_goose/unstable/session/info": { @@ -995,13 +994,12 @@ export function buildInitScript(options?: { sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, - cwd: session.cwd, + cwd: session.cwd, messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, providerId: session.providerId, modelId: session.modelId, conversationBefore: session.conversationBefore, - cwd: session.workingDir ?? "/tmp/alpha", })), ); case "create_session": From 757b78a9711576cb44789618f462c9786694dba0 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 21:30:20 -0400 Subject: [PATCH 18/19] docs(voice): explain pending transcript delivery in settings Signed-off-by: John Matthew Tennant --- src/shared/i18n/locales/en/settings.json | 2 +- src/shared/i18n/locales/es/settings.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index ca4cc32fd..350a60a30 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1077,7 +1077,7 @@ "voiceLabel": "Pocket TTS voice", "earlyDeliverySaveFailed": "Could not save early delivery setting.", "earlyDelivery": "Early delivery (experimental)", - "earlyDeliveryDescription": "Prepare a private draft while you speak to reduce response time. Your final transcript remains authoritative.", + "earlyDeliveryDescription": "Send pending transcripts to the agent while you speak, so it can start preparing a response. Only your final transcript can trigger actions.", "statusSounds": "Status sounds", "statusSoundModes": { "off": "Off", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index fbed65f6e..c4305fb1c 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1076,7 +1076,7 @@ "voiceLabel": "Voz de Pocket TTS", "earlyDeliverySaveFailed": "No se pudo guardar la configuración de entrega anticipada.", "earlyDelivery": "Entrega anticipada (experimental)", - "earlyDeliveryDescription": "Prepara un borrador privado mientras hablas para reducir el tiempo de respuesta. La transcripción final sigue siendo la definitiva.", + "earlyDeliveryDescription": "Envía transcripciones pendientes al agente mientras hablas para que pueda empezar a preparar una respuesta. Solo la transcripción final puede activar acciones.", "statusSounds": "Sonidos de estado", "statusSoundModes": { "off": "Desactivados", From 73f6436da94e18bc648e214755b12f97ec41e927 Mon Sep 17 00:00:00 2001 From: John Matthew Tennant Date: Thu, 29 Oct 2026 22:00:15 -0400 Subject: [PATCH 19/19] fix(voice): prepare from interim transcripts during active speech Signed-off-by: John Matthew Tennant --- src-tauri/src/commands/native_voice.rs | 39 ++++++++++++++------------ tests/e2e/voice-conversation.spec.ts | 10 +++++++ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index fb0e1a8f0..af3d5eefc 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1455,13 +1455,12 @@ fn reject_pending_transcript( struct EarlyVoiceTranscript { utterance_id: String, interim_text: String, - user_speaking: bool, early_emitted: bool, } impl EarlyVoiceTranscript { fn take_ready(&mut self) -> bool { - if self.user_speaking || self.early_emitted || self.interim_text.is_empty() { + if self.early_emitted || self.interim_text.is_empty() { return false; } self.early_emitted = true; @@ -1809,20 +1808,7 @@ pub async fn start_native_voice_conversation( log::warn!("Voice input emitted duplicate readiness"); } berd_call::input::VoiceInputEvent::SpeakingChanged(speaking) => { - early.user_speaking = speaking; admission.set_user_speaking(speaking); - if early.take_ready() { - let _ = event_window.emit( - EVENT_NAME, - NativeVoiceEvent::InterimUser { - session_id: session_id.clone(), - lifecycle_id: lifecycle_id.clone(), - id: early.utterance_id.clone(), - text: early.interim_text.clone(), - revision, - }, - ); - } event_state.set_status_sound_input_activity(&session_id, revision, speaking); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), @@ -2856,13 +2842,30 @@ mod tests { assert_eq!(new_final, new_provisional); } + #[test] + fn speaking_interim_is_delivered_before_final_and_vad_end() { + let mut early = EarlyVoiceTranscript::default(); + early.set_interim("recognition".into(), " first words ".into()); + assert!( + early.take_ready(), + "private preparation must start while speaking" + ); + assert_eq!(early.interim_text, "first words"); + early.set_interim("recognition".into(), "first words continued".into()); + assert!(!early.take_ready(), "one draft per recognition segment"); + // The final can arrive before VAD ends. Its private draft already started. + early.finish("recognition"); + assert!(!early.take_ready()); + early.set_interim("next-recognition".into(), " ".into()); + assert!(!early.take_ready(), "blank interims must not start drafts"); + early.set_interim("next-recognition".into(), "next words".into()); + assert!(early.take_ready()); + } + #[test] fn recognition_resolution_allows_early_delivery_after_no_final_or_mute() { let mut early = EarlyVoiceTranscript::default(); - early.user_speaking = true; early.set_interim("first-recognition".into(), "first words".into()); - assert!(!early.take_ready()); - early.user_speaking = false; assert!(early.take_ready()); assert!(!early.take_ready()); let previous_id = early.utterance_id.clone(); diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index c8dce6716..b241f845c 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -417,6 +417,10 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { "An unrelated running answer.", "unrelated-run", ); + await nativeEvent(page, { + type: "activity", + activity: "user-speaking", + }); await nativeEvent(page, { type: "interimUser", ...utterance }); const allRequests = () => page.evaluate(() => @@ -482,6 +486,8 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { const finalText = busy ? utterance.text : "Set the temperature to -10 degrees."; + // The private draft has started while speaking. The final arrives + // before VAD idle, and remains the sole authoritative user turn. const started = Date.now(); await nativeEvent(page, { type: "user", @@ -497,6 +503,10 @@ for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { .poll(mainRequests, { timeout: 1800, intervals: [20, 50] }) .toHaveLength(1); expect(Date.now() - started).toBeLessThan(1800); + await nativeEvent(page, { + type: "activity", + activity: "user-idle", + }); const final = (await mainRequests())[0]; expect(final.method).toBe( busy ? "_goose/unstable/session/steer" : "session/prompt",