diff --git a/src-tauri/crates/berd-call/src/benchmark/stt.rs b/src-tauri/crates/berd-call/src/benchmark/stt.rs index 1f4c7de50..0cd6c992c 100644 --- a/src-tauri/crates/berd-call/src/benchmark/stt.rs +++ b/src-tauri/crates/berd-call/src/benchmark/stt.rs @@ -563,7 +563,7 @@ fn observe_event( tracker.speaking_ended = Some(now); } } - Some(VoiceInputEvent::InterimTranscript(_)) => {} + Some(VoiceInputEvent::InterimTranscript { .. }) => {} Some(VoiceInputEvent::RecognitionPendingChanged(active)) => { tracker.pending = active; if active { @@ -576,6 +576,7 @@ fn observe_event( Some(VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. }) => { if tracker.final_text.is_some() { tracker.error = @@ -1044,6 +1045,7 @@ mod tests { let (first_receipt, first_ack) = crate::input::FinalTranscriptStorageReceipt::test_pair(); observe_event( Some(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "stored words".into(), storage_receipt: first_receipt, }), @@ -1058,6 +1060,7 @@ mod tests { crate::input::FinalTranscriptStorageReceipt::test_pair(); assert!(observe_event( Some(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "duplicate words".into(), storage_receipt: duplicate_receipt, }), @@ -1076,6 +1079,7 @@ mod tests { fn late_final_is_a_shutdown_failure_and_is_not_acknowledged() { let (receipt, acknowledgement) = crate::input::FinalTranscriptStorageReceipt::test_pair(); let error = observe_shutdown_event(VoiceInputEvent::FinalTranscript { + recognition_id: "recognition".into(), text: "late words".into(), storage_receipt: receipt, }) diff --git a/src-tauri/crates/berd-call/src/input.rs b/src-tauri/crates/berd-call/src/input.rs index 52fb576b7..f8bdfee61 100644 --- a/src-tauri/crates/berd-call/src/input.rs +++ b/src-tauri/crates/berd-call/src/input.rs @@ -124,10 +124,13 @@ pub enum VoiceInputEvent { SpeakingChanged(bool), /// Agent speech admission hold, which may expire before recognition finishes. RecognitionPendingChanged(bool), - /// Display only: words Apple is still transcribing, or empty once none are. - /// Never stored, delivered or treated as an utterance. - InterimTranscript(String), + /// Provisional words, identified by their pending recognition rather than its admission hold. + InterimTranscript { + recognition_id: String, + text: String, + }, FinalTranscript { + recognition_id: String, text: String, storage_receipt: FinalTranscriptStorageReceipt, }, @@ -726,6 +729,10 @@ struct PendingRecognitions { unfinalized: bool, /// The interim text last shown, so repeats and clears are sent once. interim: String, + interim_recognition_id: String, + /// Apple recognition segments can span multiple VAD admission holds. + #[cfg(any(target_os = "macos", test))] + segment_id: Option, } impl PendingRecognitions { @@ -795,11 +802,43 @@ impl PendingRecognitions { self.unfinalized = false; } + /// Apple reports cumulative text until a final, independent of VAD pauses. + #[cfg(any(target_os = "macos", test))] + fn show_segment_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { + let id = self + .segment_id + .get_or_insert_with(|| uuid::Uuid::new_v4().to_string()) + .clone(); + self.emit_interim(events, text, id); + } + + #[cfg(any(target_os = "macos", test))] + fn take_segment_recognition_id(&mut self) -> String { + self.segment_id + .take() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()) + } + + fn clear_interim(&mut self, events: &tokio_mpsc::Sender) { + self.emit_interim(events, "", String::new()); + } + /// Interim text is best effort: a full event queue drops it rather than stall recognition. - fn show_interim(&mut self, events: &tokio_mpsc::Sender, text: &str) { - if self.interim != text { + fn emit_interim( + &mut self, + events: &tokio_mpsc::Sender, + text: &str, + recognition_id: String, + ) { + if self.interim != text + || (!text.is_empty() && self.interim_recognition_id != recognition_id) + { self.interim = text.to_string(); - let _ = events.try_send(VoiceInputEvent::InterimTranscript(self.interim.clone())); + self.interim_recognition_id = recognition_id.clone(); + let _ = events.try_send(VoiceInputEvent::InterimTranscript { + recognition_id, + text: self.interim.clone(), + }); } } @@ -837,7 +876,7 @@ impl PendingRecognitions { fn reset(&mut self, events: &tokio_mpsc::Sender) -> Result<(), ()> { self.release_hold(events)?; - self.show_interim(events, ""); + self.clear_interim(events); *self = Self::new(); Ok(()) } @@ -887,6 +926,7 @@ fn clamp_vad_frame(samples: &[f32]) -> Vec { fn send_final( events: &tokio_mpsc::Sender, + recognition_id: String, text: String, storage_deadline: Option, ) -> Result<(), ()> { @@ -898,6 +938,7 @@ fn send_final( }; events .blocking_send(VoiceInputEvent::FinalTranscript { + recognition_id, text, storage_receipt: receipt, }) @@ -1488,7 +1529,12 @@ fn complete_parakeet_recognition( storage_deadline: Option, ) { if !text.is_empty() { - let _ = send_final(events, text, storage_deadline); + let _ = send_final( + events, + uuid::Uuid::new_v4().to_string(), + text, + storage_deadline, + ); } let _ = pending.resolve(events); } @@ -1520,8 +1566,9 @@ fn forward_mac_events( match event { crate::mac_speech::MacSpeechRecognitionEvent::Final(text, end) => { let text = text.trim().to_string(); + let recognition_id = pending.take_segment_recognition_id(); if !text.is_empty() { - send_final(events, text, storage_deadline)?; + send_final(events, recognition_id, text, storage_deadline)?; } // One final can span several turns split by pauses, and one turn // can produce several finals, so a final with an end resolves @@ -1535,12 +1582,12 @@ fn forward_mac_events( *settle_deadline = None; } if !pending.words_outstanding() { - pending.show_interim(events, ""); + pending.clear_interim(events); } } crate::mac_speech::MacSpeechRecognitionEvent::Heard(_, text) => { pending.unfinalized = true; - pending.show_interim(events, &text); + pending.show_segment_interim(events, &text); } crate::mac_speech::MacSpeechRecognitionEvent::Finished => { if storage_deadline.is_none() { @@ -1669,7 +1716,7 @@ fn mac_speech_worker( return; } pending.abandon_heard_words(); - pending.show_interim(&events, ""); + pending.clear_interim(&events); settle_deadline = None; } let frame = match frames.recv_timeout(Duration::from_millis(50)) { @@ -1946,7 +1993,12 @@ fn deliver_openai_turns( .then_some(final_storage_deadline) .flatten(); if !text.is_empty() { - let _ = send_final(events, text, storage_deadline); + let _ = send_final( + events, + uuid::Uuid::new_v4().to_string(), + text, + storage_deadline, + ); } let _ = pending.resolve(events); } @@ -2760,12 +2812,13 @@ mod tests { VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => { assert_eq!(text, "fixture final"); storage_receipt.stored(); observed.push("final"); } - VoiceInputEvent::InterimTranscript(_) => {} + VoiceInputEvent::InterimTranscript { .. } => {} VoiceInputEvent::Failed(error) => panic!("unexpected input failure: {error}"), } } @@ -3103,6 +3156,81 @@ mod tests { ); } + #[test] + fn apple_cumulative_segment_survives_pause_turns() { + let (events, mut receiver) = tokio_mpsc::channel(16); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.show_segment_interim(&events, "first words"); + pending.end_turn(&events, 100).unwrap(); + pending.begin(&events).unwrap(); + pending.show_segment_interim(&events, "first words and more"); + let mut ids = Vec::new(); + while let Ok(event) = receiver.try_recv() { + if let VoiceInputEvent::InterimTranscript { recognition_id, .. } = event { + ids.push(recognition_id); + } + } + assert_eq!(ids[0], ids[1], "one Apple segment across two VAD turns"); + pending.release_hold(&events).unwrap(); + assert_eq!(pending.take_segment_recognition_id(), ids[1]); + pending.show_segment_interim(&events, "next segment"); + assert_ne!(pending.take_segment_recognition_id(), ids[1]); + } + + #[test] + fn final_only_backends_generate_distinct_recognition_ids() { + for openai in [false, true] { + let (events, mut receiver) = tokio_mpsc::channel(16); + let mut pending = PendingRecognitions::new(); + pending.begin(&events).unwrap(); + pending.release_hold(&events).unwrap(); + pending.begin(&events).unwrap(); + for text in ["old final", "new final"] { + if openai { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: text.into(), + mute_epoch: 0, + settle_deadline: Instant::now(), + }]); + let mut completed = HashMap::from([(text.into(), text.into())]); + deliver_openai_turns( + &mut committed, + &mut completed, + &events, + &mut pending, + None, + None, + ); + } else { + complete_parakeet_recognition(text.into(), &events, &mut pending, None); + } + assert_eq!(pending.blocking, usize::from(text == "old final")); + } + let mut finals = Vec::new(); + while let Ok(event) = receiver.try_recv() { + if let VoiceInputEvent::FinalTranscript { + recognition_id, + text, + .. + } = event + { + assert_eq!( + uuid::Uuid::parse_str(&recognition_id) + .unwrap() + .get_version_num(), + 4 + ); + finals.push((recognition_id, text)); + } + } + assert_eq!(finals.len(), 2); + assert_ne!(finals[0].0, finals[1].0); + assert_eq!(finals[0].1, "old final"); + assert_eq!(finals[1].1, "new final"); + } + } + #[test] fn final_storage_receipt_unblocks_the_bounded_sender() { let (events, mut receiver) = tokio_mpsc::channel(1); @@ -3110,6 +3238,7 @@ mod tests { let sender = thread::spawn(move || { send_final( &events, + "recognition".into(), "stored words".to_string(), Some(Instant::now() + Duration::from_secs(1)), ) @@ -3119,6 +3248,7 @@ mod tests { let VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } = receiver.blocking_recv().expect("final event") else { panic!("expected final transcript") @@ -3139,6 +3269,7 @@ mod tests { let sender = thread::spawn(move || { send_final( &events, + "recognition".into(), "unstored words".to_string(), Some(Instant::now() + Duration::from_secs(1)), ) @@ -3166,8 +3297,20 @@ mod tests { let (done_tx, done_rx) = mpsc::sync_channel(0); let deadline = Instant::now() + Duration::from_millis(100); let sender = thread::spawn(move || { - send_final(&events, "first".to_string(), Some(deadline)).unwrap(); - send_final(&events, "second".to_string(), Some(deadline)).unwrap(); + send_final( + &events, + "first-id".into(), + "first".to_string(), + Some(deadline), + ) + .unwrap(); + send_final( + &events, + "second-id".into(), + "second".to_string(), + Some(deadline), + ) + .unwrap(); done_tx.send(()).unwrap(); }); let VoiceInputEvent::FinalTranscript { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 4de0b2203..fc2a0ce8a 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -4339,7 +4339,7 @@ fn handle_voice_input_event( )?; update_provisional_suspension(core, active)?; } - VoiceInputEvent::InterimTranscript(text) => write_message( + VoiceInputEvent::InterimTranscript { text, .. } => write_message( writer, &SessionMessage::UserInterim { token: *next_token, @@ -4349,6 +4349,7 @@ fn handle_voice_input_event( VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => store_and_publish_voice_final( text, || storage_receipt.stored(), @@ -8182,7 +8183,10 @@ mod tests { let mut output = Vec::new(); let mut next_token = 7; handle_voice_input_event( - VoiceInputEvent::InterimTranscript("you sh".into()), + VoiceInputEvent::InterimTranscript { + recognition_id: "recognition".into(), + text: "you sh".into(), + }, &mut core, &mut active, &mut next_token, diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 0ac09411d..af3d5eefc 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -205,6 +205,18 @@ pub(crate) enum NativeVoiceEvent { revision: u64, delivery_attempts: u8, }, + RecognitionPending { + session_id: String, + pending: bool, + revision: u64, + }, + InterimUser { + session_id: String, + lifecycle_id: String, + id: String, + text: String, + revision: u64, + }, Activity { session_id: String, activity: &'static str, @@ -1439,6 +1451,43 @@ fn reject_pending_transcript( Ok(TranscriptRejection { attempts, terminal }) } +#[derive(Default)] +struct EarlyVoiceTranscript { + utterance_id: String, + interim_text: String, + early_emitted: bool, +} + +impl EarlyVoiceTranscript { + fn take_ready(&mut self) -> bool { + if self.early_emitted || self.interim_text.is_empty() { + return false; + } + self.early_emitted = true; + true + } + + fn set_interim(&mut self, recognition_id: String, text: String) { + if self.utterance_id != recognition_id { + self.reset(); + self.utterance_id = recognition_id; + } + self.interim_text = text.trim().to_string(); + } + + fn finish(&mut self, recognition_id: &str) { + if self.utterance_id == recognition_id { + self.reset(); + } + } + + fn reset(&mut self) { + self.utterance_id.clear(); + self.interim_text.clear(); + self.early_emitted = false; + } +} + #[tauri::command] #[allow(clippy::too_many_arguments)] // Tauri injects four guards beside the lifecycle claim. pub async fn start_native_voice_conversation( @@ -1744,6 +1793,7 @@ pub async fn start_native_voice_conversation( .ok_or_else(|| "Voice admission was not initialized.".to_string())?; let event_state = state.inner().clone(); tauri::async_runtime::spawn(async move { + let mut early = EarlyVoiceTranscript::default(); while let Some(event) = events.recv().await { let active = runtime.lock().ok().is_some_and(|current| { current.session_id.as_deref() == Some(session_id.as_str()) @@ -1774,23 +1824,50 @@ pub async fn start_native_voice_conversation( } berd_call::input::VoiceInputEvent::RecognitionPendingChanged(pending) => { admission.set_recognition_pending(pending); - // The runtime owns recognition-pending sequencing. Berd's - // renderer does not project that state yet. + if !pending { + early.reset(); + } + let _ = event_window.emit( + EVENT_NAME, + NativeVoiceEvent::RecognitionPending { + session_id: session_id.clone(), + pending, + revision, + }, + ); + } + berd_call::input::VoiceInputEvent::InterimTranscript { + recognition_id, + text, + } => { + early.set_interim(recognition_id, text); + if early.take_ready() { + let _ = event_window.emit( + EVENT_NAME, + NativeVoiceEvent::InterimUser { + session_id: session_id.clone(), + lifecycle_id: lifecycle_id.clone(), + id: early.utterance_id.clone(), + text: early.interim_text.clone(), + revision, + }, + ); + } } - // Berd's renderer does not show interim words yet. - berd_call::input::VoiceInputEvent::InterimTranscript(_) => {} berd_call::input::VoiceInputEvent::FinalTranscript { + recognition_id, text, storage_receipt, } => { let transcript = PendingTranscript { session_id: session_id.clone(), lifecycle_id: lifecycle_id.clone(), - id: uuid::Uuid::new_v4().to_string(), + id: recognition_id.clone(), text, revision, delivery_attempts: 0, }; + early.finish(&recognition_id); let Ok(disposition) = store_final_if_active( &runtime, &pending, @@ -2739,6 +2816,66 @@ mod tests { use super::*; use std::{thread, time::Instant}; + #[test] + fn late_final_keeps_identity_after_hold_expires_and_new_provisional() { + let mut early = EarlyVoiceTranscript::default(); + early.set_interim("old-recognition".into(), "old words".into()); + assert!(early.take_ready()); + let old_provisional = early.utterance_id.clone(); + // RecognitionPendingChanged(false) expires the hold, not recognition. + early.reset(); + early.set_interim("new-recognition".into(), "new words".into()); + assert!(early.take_ready()); + let new_provisional = early.utterance_id.clone(); + assert_ne!(old_provisional, new_provisional); + // The old final arrives while the newer provisional is active. + let old_final = "old-recognition"; + early.finish(old_final); + assert_eq!(old_final, old_provisional); + assert_eq!(early.utterance_id, new_provisional); + assert!( + !early.take_ready(), + "old final must not re-emit the new provisional" + ); + let new_final = "new-recognition"; + early.finish(new_final); + assert_eq!(new_final, new_provisional); + } + + #[test] + fn speaking_interim_is_delivered_before_final_and_vad_end() { + let mut early = EarlyVoiceTranscript::default(); + early.set_interim("recognition".into(), " first words ".into()); + assert!( + early.take_ready(), + "private preparation must start while speaking" + ); + assert_eq!(early.interim_text, "first words"); + early.set_interim("recognition".into(), "first words continued".into()); + assert!(!early.take_ready(), "one draft per recognition segment"); + // The final can arrive before VAD ends. Its private draft already started. + early.finish("recognition"); + assert!(!early.take_ready()); + early.set_interim("next-recognition".into(), " ".into()); + assert!(!early.take_ready(), "blank interims must not start drafts"); + early.set_interim("next-recognition".into(), "next words".into()); + assert!(early.take_ready()); + } + + #[test] + fn recognition_resolution_allows_early_delivery_after_no_final_or_mute() { + let mut early = EarlyVoiceTranscript::default(); + early.set_interim("first-recognition".into(), "first words".into()); + assert!(early.take_ready()); + assert!(!early.take_ready()); + let previous_id = early.utterance_id.clone(); + early.reset(); + assert_ne!(early.utterance_id, previous_id); + assert!(!early.take_ready()); + early.set_interim("next-recognition".into(), "next words".into()); + assert!(early.take_ready()); + } + fn pending_transcript( session_id: &str, lifecycle_id: &str, @@ -3496,6 +3633,7 @@ mod tests { VoiceInputEvent::FinalTranscript { text, storage_receipt, + .. } => { storage_receipt.stored(); if text == "marker" { diff --git a/src/features/experiments/experimentDefinitions.ts b/src/features/experiments/experimentDefinitions.ts index 54bccb0bb..1916a1386 100644 --- a/src/features/experiments/experimentDefinitions.ts +++ b/src/features/experiments/experimentDefinitions.ts @@ -100,6 +100,14 @@ export const EXPERIMENT_DEFINITIONS = [ titleKey: "experiments.voiceConversation.title", descriptionKey: "experiments.voiceConversation.description", defaultEnabled: true, + config: { + earlyDelivery: { + type: "boolean", + labelKey: "voice.earlyDelivery", + descriptionKey: "voice.earlyDeliveryDescription", + defaultValue: true, + }, + }, }, { id: CHAT_ON_CANVAS_EXPERIMENT_ID, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index a80850283..86401bd85 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -227,6 +227,20 @@ export type VoiceConversationEvent = revision: number; deliveryAttempts: number; } + | { + type: "recognitionPending"; + sessionId: string; + pending: boolean; + revision: number; + } + | { + type: "interimUser"; + sessionId: string; + lifecycleId: string; + id: string; + text: string; + revision: number; + } | { type: "activity"; sessionId: string; diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 6752807b8..e372fedae 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,8 +3,17 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import { VoiceMicrophoneCaptureError } from "../api/voiceConversation"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { setVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { setStatusSoundPreference } from "../lib/statusSoundPreference"; +const privateDraftMocks = vi.hoisted(() => ({ + start: vi.fn(), +})); +vi.mock("../lib/privateVoiceDraft", () => ({ + startPrivateVoiceDraft: privateDraftMocks.start, +})); + const nativeAssistantSpeechMocks = vi.hoisted(() => ({ capture: vi.fn(() => []), start: vi.fn(), @@ -319,6 +328,78 @@ describe("voice transcript delivery coordination", () => { useChatStore.setState({ messagesBySession: {}, sessionStateById: {} }); }); + it("gates preparation, disposes outstanding drafts on disable, and still delivers finals", async () => { + const dispose = vi.fn(); + const take = vi.fn().mockResolvedValue("private draft"); + privateDraftMocks.start.mockReset(); + privateDraftMocks.start.mockReturnValue({ dispose, take }); + const onSend = vi.fn().mockResolvedValue(true); + useChatSessionStore.setState({ + sessions: [{ id: "session-1", workingDir: "/project" }] as never, + }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 42, + }, + microphoneMuted: false, + uiState: "listening", + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + }); + const control = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend, + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + const event = { + sessionId: "session-1", + lifecycleId: "early-toggle", + id: "toggle-utterance", + text: "partial", + revision: 42, + deliveryAttempts: 0, + }; + await voiceStoreMocks.subscriber?.({ ...event, type: "interimUser" }); + expect(privateDraftMocks.start).toHaveBeenCalledOnce(); + act(() => setVoiceEarlyDeliveryPreference({ enabled: false })); + expect(dispose).toHaveBeenCalledOnce(); + await voiceStoreMocks.subscriber?.({ + ...event, + id: "disabled-utterance", + type: "interimUser", + }); + expect(privateDraftMocks.start).toHaveBeenCalledOnce(); + await act(async () => { + await voiceStoreMocks.subscriber?.({ + ...event, + text: "authoritative final", + type: "user", + }); + }); + expect(take).not.toHaveBeenCalled(); + expect(onSend).toHaveBeenCalledWith( + "authoritative final", + undefined, + undefined, + expect.objectContaining({ + displayText: "authoritative final", + assistantPrompt: "", + }), + ); + control.unmount(); + }); + it("applies settings changes to an active chained runtime", async () => { useVoiceConversationStore.setState({ status: { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 4a5cb37bb..37fac1d9d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -6,6 +6,15 @@ import type { ChatInputSendHandler, ChatInputVoiceConversation, } from "@/features/chat/types"; +import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; +import { + startPrivateVoiceDraft, + type PrivateVoiceDraft, +} from "../lib/privateVoiceDraft"; +import { + getVoiceEarlyDeliveryPreference, + subscribeToVoiceEarlyDeliveryPreference, +} from "../lib/voiceEarlyDeliveryPreference"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; import { createSystemNotificationMessage } from "@/shared/types/messages"; @@ -524,11 +533,59 @@ export function resetVoiceUiWhenRunSettles( queueMicrotask(check); } +const earlyDeliveries = new Map(); +const finalTranscriptsSeen = new Set(); + +function voiceEventKey(event: { + sessionId: string; + lifecycleId: string; + revision: number; + id: string; +}): string { + return `${event.sessionId}\0${event.lifecycleId}\0${event.revision}\0${event.id}`; +} + +function earlyEventIsCurrent( + event: { sessionId: string; revision: number }, + requireMicrophone = true, +) { + const { status, microphoneMuted } = useVoiceConversationStore.getState(); + return ( + status.lifecycle === "running" && + status.sessionId === event.sessionId && + status.revision === event.revision && + status.ownerWindowLabel === getCurrentWindow().label && + (!requireMicrophone || !microphoneMuted) + ); +} + function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; observeChainedVoiceStatus(); - subscribeToVoiceConversationEvents(async (event) => { + subscribeToVoiceEarlyDeliveryPreference(() => { + if (getVoiceEarlyDeliveryPreference().enabled) return; + for (const draft of earlyDeliveries.values()) draft.dispose(); + earlyDeliveries.clear(); + finalTranscriptsSeen.clear(); + }); + useVoiceConversationStore.subscribe((state, previous) => { + if ( + state.status.revision === previous.status.revision && + state.status.sessionId === previous.status.sessionId && + state.status.lifecycle === previous.status.lifecycle + ) + return; + for (const draft of earlyDeliveries.values()) draft.dispose(); + earlyDeliveries.clear(); + finalTranscriptsSeen.clear(); + }); + subscribeToVoiceConversationEvents(async (incoming) => { + const isEarly = incoming.type === "interimUser"; + const event = + incoming.type === "interimUser" + ? { ...incoming, type: "user" as const, deliveryAttempts: 0 } + : incoming; if (event.type === "cleanShutdown" || event.type === "controlsDismissed") { return; } @@ -542,6 +599,16 @@ function ensureVoiceEventDeliveryInitialized() { addErrorNotification(sessionId ?? null, event.message); return; } + if (event.type === "recognitionPending") { + if (event.pending) return; + if (!earlyEventIsCurrent(event, false)) return; + for (const [key, draft] of earlyDeliveries) { + if (finalTranscriptsSeen.has(key)) continue; + draft.dispose(); + earlyDeliveries.delete(key); + } + return; + } if (event.type === "activity") return; if (event.type !== "user" || !event.text.trim()) return; if ( @@ -555,9 +622,37 @@ function ensureVoiceEventDeliveryInitialized() { return; } + const eventKey = voiceEventKey(event); const deliveryRevision = event.revision; const shouldNotifyFailure = event.deliveryAttempts === 0; + if (isEarly) { + if ( + !getVoiceEarlyDeliveryPreference().enabled || + !earlyEventIsCurrent(event) || + earlyDeliveries.has(eventKey) || + activeRouteIsBlocked(event.sessionId) || + activeSendRoute?.sessionId !== event.sessionId + ) + return; + const session = useChatSessionStore + .getState() + .sessions.find((session) => session.id === event.sessionId); + if (!session?.workingDir) return; + for (const [key, draft] of earlyDeliveries) { + if (finalTranscriptsSeen.has(key)) continue; + draft.dispose(); + earlyDeliveries.delete(key); + } + // Independent of the authoritative queue, even when the main run is busy. + earlyDeliveries.set( + eventKey, + startPrivateVoiceDraft(event.sessionId, session.workingDir, event.text), + ); + return; + } + if (earlyDeliveries.has(eventKey)) finalTranscriptsSeen.add(eventKey); return enqueueVoiceTranscriptDelivery(event.sessionId, async () => { + const early = earlyDeliveries.get(eventKey); if (activeRouteIsBlocked(event.sessionId)) { throw new VoiceTranscriptDeferredError( "Voice transcript is waiting for its bound chat to become available.", @@ -574,9 +669,12 @@ function ensureVoiceEventDeliveryInitialized() { const store = useVoiceConversationStore.getState(); store.setUiState("user-speaking"); - + const draft = await early?.take(); + earlyDeliveries.delete(eventKey); + finalTranscriptsSeen.delete(eventKey); const sendOptions = { userMessageMetadata: { + userVisible: true, origin: "voice_conversation" as const, voiceUtteranceId: event.id, voiceConversationLifecycleId: event.lifecycleId, @@ -590,8 +688,6 @@ function ensureVoiceEventDeliveryInitialized() { }, }; try { - // This runs inside the per-session queue, so a prior send can change - // the opportunity to steer before the next transcript is evaluated. const opportunity = await waitForVoiceDeliveryOpportunity( event.sessionId, ); @@ -609,7 +705,14 @@ function ensureVoiceEventDeliveryInitialized() { const playbackNotice = takeVoicePlaybackNotices(event.sessionId); const displayOptions = { ...sendOptions, - ...(playbackNotice ? { assistantPrompt: playbackNotice } : {}), + assistantPrompt: [ + playbackNotice, + draft && getVoiceEarlyDeliveryPreference().enabled + ? `Private provisional DRAFT (not spoken, not executed, may be wrong):\n${draft}\n\nThe user transcript in this turn is authoritative. Answer it and perform any requested actions normally. Use the draft only if it is relevant to the final transcript.` + : null, + ] + .filter(Boolean) + .join("\n\n"), displayText: event.text, }; store.setUiState("agent-working"); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 546ba03d5..7ea41ce9d 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -1432,6 +1432,7 @@ export function startNativeAssistantSpeech( return; } const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; + cacheCausalTranscriptKeys(messages); if ( heldSpeech || voice.userSpeaking || diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.test.ts b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts new file mode 100644 index 000000000..f1ecbb7e9 --- /dev/null +++ b/src/features/voice-conversation/lib/privateVoiceDraft.test.ts @@ -0,0 +1,144 @@ +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { startPrivateVoiceDraft } from "./privateVoiceDraft"; +const mocks = vi.hoisted(() => ({ + forkSession: vi.fn(), + newSession: vi.fn(), + getSessionInfo: vi.fn(), + setModel: vi.fn(), + setSessionMode: vi.fn(), + prompt: vi.fn(), + cancelSession: vi.fn(), + deleteSession: vi.fn(), + hide: vi.fn(), + intercept: vi.fn(), +})); +vi.mock("@/shared/api/acpApi", () => mocks); +vi.mock("@/shared/api/acpConnection", () => ({ + hidePrivateSessionNotifications: mocks.hide, + interceptSessionNotifications: mocks.intercept, +})); +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} +beforeEach(() => { + vi.useFakeTimers(); + vi.resetAllMocks(); + mocks.getSessionInfo.mockResolvedValue({ + providerId: "openai", + modelId: "configured-model", + }); + mocks.newSession.mockResolvedValue({ sessionId: "private" }); + mocks.setModel.mockResolvedValue(undefined); + mocks.setSessionMode.mockResolvedValue(undefined); + mocks.prompt.mockResolvedValue(undefined); + mocks.cancelSession.mockResolvedValue(undefined); + mocks.deleteSession.mockResolvedValue(undefined); + mocks.intercept.mockReturnValue(vi.fn()); +}); +afterEach(() => vi.useRealTimers()); +it("awaits chat mode before first prompt and captures only private text", async () => { + const mode = deferred(); + mocks.setSessionMode.mockReturnValue(mode.promise); + mocks.prompt.mockImplementation(async () => { + mocks.intercept.mock.calls[0][0]({ + sessionId: "private", + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "Draft" }, + }, + }); + }); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.setSessionMode).toHaveBeenCalledWith("private", "chat"); + expect(mocks.prompt).not.toHaveBeenCalled(); + mode.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(await draft.take()).toBe("Draft"); + expect(mocks.forkSession).not.toHaveBeenCalled(); + expect(mocks.getSessionInfo).toHaveBeenCalledWith("main"); + expect(mocks.newSession).toHaveBeenCalledWith("/work", { + hidden: true, + providerId: "openai", + backendId: "local", + }); + expect(mocks.setModel).toHaveBeenCalledWith("private", "configured-model"); + expect(mocks.prompt).toHaveBeenCalledWith("private", [ + { type: "text", text: "interim" }, + ]); + expect(mocks.cancelSession).toHaveBeenCalledWith("private"); +}); +it("fails closed when chat mode is rejected", async () => { + mocks.setSessionMode.mockRejectedValue(new Error("unsupported")); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await expect(draft.take()).resolves.toBeNull(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it.each([ + "creation", + "mode", + "prompt", +])("bounds stalled %s and cancellation without touching main", async (stage) => { + const stalled = deferred(); + mocks[ + stage === "creation" + ? "newSession" + : stage === "mode" + ? "setSessionMode" + : "prompt" + ].mockReturnValue(stalled.promise); + mocks.cancelSession.mockReturnValue(stalled.promise); + mocks.deleteSession.mockReturnValue(stalled.promise); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + const taken = draft.take(); + await vi.advanceTimersByTimeAsync(750); + expect(await taken).toBeNull(); + expect(mocks.cancelSession.mock.calls.every(([id]) => id === "private")).toBe( + true, + ); + if (stage !== "prompt") expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("bounds best effort setup and cleans a private session created after disposal", async () => { + const creation = deferred<{ sessionId: string }>(); + mocks.newSession.mockReturnValue(creation.promise); + startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(2000); + creation.resolve({ sessionId: "late-private" }); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.cancelSession).toHaveBeenCalledWith("late-private"); + expect(mocks.hide).toHaveBeenCalledWith("late-private"); + expect(mocks.setSessionMode).not.toHaveBeenCalled(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("never prompts after a timed out mode ack", async () => { + const mode = deferred(); + mocks.setSessionMode.mockReturnValue(mode.promise); + startPrivateVoiceDraft("main", "/work", "interim"); + await vi.advanceTimersByTimeAsync(2000); + mode.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); + +it.each([ + { providerId: null, modelId: "model" }, + { providerId: "openai", modelId: null }, +])("fails closed without configured provider/model: %o", async (config) => { + mocks.getSessionInfo.mockResolvedValue(config); + const draft = startPrivateVoiceDraft("main", "/work", "interim"); + await expect(draft.take()).resolves.toBeNull(); + expect(mocks.newSession).not.toHaveBeenCalled(); + expect(mocks.forkSession).not.toHaveBeenCalled(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); +it("fails closed on model configuration failure", async () => { + mocks.setModel.mockRejectedValue(new Error("unavailable")); + await expect( + startPrivateVoiceDraft("main", "/work", "interim").take(), + ).resolves.toBeNull(); + expect(mocks.prompt).not.toHaveBeenCalled(); +}); diff --git a/src/features/voice-conversation/lib/privateVoiceDraft.ts b/src/features/voice-conversation/lib/privateVoiceDraft.ts new file mode 100644 index 000000000..ec90475e6 --- /dev/null +++ b/src/features/voice-conversation/lib/privateVoiceDraft.ts @@ -0,0 +1,108 @@ +import { + cancelSession, + deleteSession, + getSessionInfo, + newSession, + setModel, + prompt, + setSessionMode, +} from "@/shared/api/acpApi"; +import { + hidePrivateSessionNotifications, + interceptSessionNotifications, +} from "@/shared/api/acpConnection"; + +import { getSessionBackend } from "@/shared/api/acpSessionBackends"; + +export interface PrivateVoiceDraft { + take: () => Promise; + dispose: () => void; +} + +/** No main-session run, mode, permission or notification state is changed. */ +export function startPrivateVoiceDraft( + sourceSessionId: string, + workingDir: string, + text: string, +): PrivateVoiceDraft { + let stopped = false; + let privateSessionId: string | undefined; + let stopIntercepting: (() => void) | undefined; + let setupTimer: ReturnType | undefined; + const cleanupSession = (sessionId: string) => { + hidePrivateSessionNotifications(sessionId); + // Neither cancellation nor deletion is trusted to settle. They cannot hold + // the final queue, and neither ever targets the source session. + void cancelSession(sessionId).catch(() => undefined); + void deleteSession(sessionId).catch(() => undefined); + }; + const dispose = () => { + if (stopped) return; + stopped = true; + clearTimeout(setupTimer); + clearTimeout(lifetimeTimer); + stopIntercepting?.(); + if (privateSessionId) cleanupSession(privateSessionId); + }; + const lifetimeTimer = setTimeout(dispose, 15_000); + setupTimer = setTimeout(dispose, 2_000); + const result = (async () => { + const source = await getSessionInfo(sourceSessionId); + if (stopped || !source.providerId?.trim() || !source.modelId?.trim()) + return null; + // Transcript-only speculation: never copy source history (including hidden + // context). The final source-session turn retains full history and authority. + // Hidden must be set at creation, not applied to a visible fork afterward. + const session = await newSession(workingDir, { + hidden: true, + providerId: source.providerId, + backendId: getSessionBackend(sourceSessionId), + }); + privateSessionId = session.sessionId; + hidePrivateSessionNotifications(privateSessionId); + if (stopped) { + cleanupSession(privateSessionId); + return null; + } + // Fail closed. Backend chat mode skips ALL tool_requests, including tools + // that bypass the permission callback. Prompt must never precede this ack. + await setModel(privateSessionId, source.modelId); + if (stopped) return null; + await setSessionMode(privateSessionId, "chat"); + if (stopped) return null; + clearTimeout(setupTimer); + const chunks: string[] = []; + stopIntercepting = interceptSessionNotifications((notification) => { + if (notification.sessionId !== privateSessionId) return false; + const { update } = notification; + if ( + !stopped && + update.sessionUpdate === "agent_message_chunk" && + update.content.type === "text" + ) + chunks.push(update.content.text); + return true; + }); + await prompt(privateSessionId, [{ type: "text", text }]); + return stopped ? null : chunks.join("").trim() || null; + })() + .catch(() => null) + .finally(dispose); + return { + dispose, + async take() { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + result, + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), 750); + }), + ]); + } finally { + clearTimeout(timer); + dispose(); + } + }, + }; +} diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts new file mode 100644 index 000000000..d3c8118f2 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.test.ts @@ -0,0 +1,85 @@ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { VOICE_CONVERSATION_EXPERIMENT_ID as id } from "@/features/experiments/experimentDefinitions"; +import { + EXPERIMENT_PREFERENCES_STORAGE_KEY as key, + getExperiment, + setExperimentConfigValue, + setExperimentEnabled, +} from "@/features/experiments/experimentPreferences"; +import { + getVoiceEarlyDeliveryPreference, + setVoiceEarlyDeliveryPreference, + useVoiceEarlyDeliveryPreference, +} from "./voiceEarlyDeliveryPreference"; + +describe("voice early delivery experiment config", () => { + beforeEach(() => window.localStorage.clear()); + it("defaults on inside enabled voice experiment", () => { + setExperimentEnabled(id, true); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + setExperimentEnabled(id, false); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); + }); + it("writes config under parent and preserves saved off", () => { + setVoiceEarlyDeliveryPreference({ enabled: false }); + expect(getExperiment(id)?.config.earlyDelivery).toBe(false); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); + }); + it("migrates explicitly saved legacy off without overwriting registry choices", () => { + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(false); + expect(getExperiment(id)?.config.earlyDelivery).toBe(false); + setExperimentConfigValue(id, "earlyDelivery", true); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + }); + it("preserves an existing registry choice over legacy off", () => { + setExperimentConfigValue(id, "earlyDelivery", true); + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + expect(getVoiceEarlyDeliveryPreference().enabled).toBe(true); + }); + it("reports failed and unsupported-version writes without losing saved off", () => { + window.localStorage.setItem(key, '{"version":999}'); + expect(setVoiceEarlyDeliveryPreference({ enabled: false })).toBe(false); + expect(window.localStorage.getItem(key)).toBe('{"version":999}'); + window.localStorage.clear(); + window.localStorage.setItem( + "goose:voice-early-delivery-preference", + '{"enabled":false}', + ); + const write = vi + .spyOn(Storage.prototype, "setItem") + .mockImplementation(() => { + throw new Error("denied"); + }); + expect(setVoiceEarlyDeliveryPreference({ enabled: false })).toBe(false); + expect( + window.localStorage.getItem("goose:voice-early-delivery-preference"), + ).toBe('{"enabled":false}'); + write.mockRestore(); + }); + it("updates same-window and cross-window subscribers", () => { + const { result } = renderHook(() => useVoiceEarlyDeliveryPreference()); + act(() => result.current.setEnabled(false)); + expect(result.current.enabled).toBe(false); + act(() => { + window.localStorage.setItem( + key, + JSON.stringify({ + version: 2, + experiments: { + [id]: { enabled: true, config: { earlyDelivery: true } }, + }, + }), + ); + window.dispatchEvent(new StorageEvent("storage", { key })); + }); + expect(result.current.enabled).toBe(true); + }); +}); diff --git a/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts new file mode 100644 index 000000000..58d262fc9 --- /dev/null +++ b/src/features/voice-conversation/lib/voiceEarlyDeliveryPreference.ts @@ -0,0 +1,70 @@ +import { useCallback } from "react"; +import { VOICE_CONVERSATION_EXPERIMENT_ID as id } from "@/features/experiments/experimentDefinitions"; +import { + EXPERIMENT_PREFERENCES_STORAGE_KEY, + getExperiment, + setExperimentConfigValue, + subscribeToExperimentChanges, + useExperiment, +} from "@/features/experiments/experimentPreferences"; + +export interface VoiceEarlyDeliveryPreference { + enabled: boolean; +} + +// Compatibility for the briefly published standalone switch. Registry choices win. +function migrateLegacyOff() { + if (typeof window === "undefined") return; + const legacyKey = "goose:voice-early-delivery-preference"; + try { + const legacy = window.localStorage.getItem(legacyKey); + if (!legacy) return; + const stored = JSON.parse( + window.localStorage.getItem(EXPERIMENT_PREFERENCES_STORAGE_KEY) ?? "{}", + ); + if ( + stored.experiments?.[id]?.config?.earlyDelivery === undefined && + JSON.parse(legacy)?.enabled === false && + !setExperimentConfigValue(id, "earlyDelivery", false) + ) + return; + window.localStorage.removeItem(legacyKey); + } catch { + // Leave failed migrations intact for the next read. + } +} + +export function getDefaultVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + return { enabled: true }; +} + +export function getVoiceEarlyDeliveryPreference(): VoiceEarlyDeliveryPreference { + migrateLegacyOff(); + const experiment = getExperiment(id); + return { + enabled: !!experiment?.enabled && experiment.config.earlyDelivery === true, + }; +} + +export const subscribeToVoiceEarlyDeliveryPreference = + subscribeToExperimentChanges; + +export function setVoiceEarlyDeliveryPreference( + preference: VoiceEarlyDeliveryPreference, +): boolean { + migrateLegacyOff(); + return setExperimentConfigValue(id, "earlyDelivery", preference.enabled); +} + +export function useVoiceEarlyDeliveryPreference() { + migrateLegacyOff(); + const experiment = useExperiment(id); + const setEnabled = useCallback( + (enabled: boolean) => setVoiceEarlyDeliveryPreference({ enabled }), + [], + ); + return { + enabled: !!experiment?.enabled && experiment.config.earlyDelivery === true, + setEnabled, + }; +} diff --git a/src/features/voice-conversation/stores/voiceConversationStore.ts b/src/features/voice-conversation/stores/voiceConversationStore.ts index 380d765dd..35ac0f11a 100644 --- a/src/features/voice-conversation/stores/voiceConversationStore.ts +++ b/src/features/voice-conversation/stores/voiceConversationStore.ts @@ -233,11 +233,11 @@ async function deliverTranscriptOnce( priorFinalizedTranscriptKeys.set(pendingKey, priorKey); } } - useVoiceConversationStore.setState((state) => - state.latestFinalizedTranscriptKey === finalizedKey + useVoiceConversationStore.setState((state) => ({ + ...(state.latestFinalizedTranscriptKey === finalizedKey ? { latestFinalizedTranscriptKey: priorKey } - : state, - ); + : {}), + })); } } return accepted ? "accepted" : deferred ? "deferred" : "rejected"; @@ -441,6 +441,7 @@ export const useVoiceConversationStore = create( revision: event.revision, }, uiState: "listening", + microphoneMuted: false, latestFinalizedTranscriptKey: null, error: null, @@ -463,6 +464,9 @@ export const useVoiceConversationStore = create( uiState: activityUiState(nextState), }; } + case "recognitionPending": + case "interimUser": + return state; case "user": return { ...state, diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 5edf6e57f..c27921a03 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -9,6 +9,7 @@ import type { MacSpeechSetup } from "../hooks/useMacSpeechSetup"; import type { SiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import type { VoiceOutputBackend } from "../lib/voiceOutputPreference"; +import { setVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { VoiceSettings } from "./VoiceSettings"; if (!HTMLElement.prototype.hasPointerCapture) { @@ -277,7 +278,32 @@ function siriSetup(): SiriVoiceSetup { } describe("VoiceSettings", () => { + it("defaults early delivery on and persists an explicit off choice", async () => { + const user = userEvent.setup(); + const view = renderWithProviders(); + const toggle = screen.getByRole("switch", { + name: "Early delivery (experimental)", + }); + expect(toggle).toBeChecked(); + await user.click(toggle); + expect(toggle).not.toBeChecked(); + view.unmount(); + renderWithProviders(); + expect( + screen.getByRole("switch", { name: "Early delivery (experimental)" }), + ).not.toBeChecked(); + }); + + it("honors saved early delivery off on initial render", () => { + setVoiceEarlyDeliveryPreference({ enabled: false }); + renderWithProviders(); + expect( + screen.getByRole("switch", { name: "Early delivery (experimental)" }), + ).not.toBeChecked(); + }); + beforeEach(async () => { + window.localStorage.clear(); await i18n.changeLanguage("en"); microphonePermissionState.status = "authorized"; microphonePermissionState.openSettingsError = false; diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index c67569fb9..14592a292 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -7,6 +7,8 @@ import { Alert, AlertDescription, AlertTitle } from "@/shared/ui/alert"; import { Button } from "@/shared/ui/button"; import { Badge } from "@/shared/ui/badge"; import { ConfirmDialog } from "@/shared/ui/confirm-dialog"; +import { Switch } from "@/shared/ui/switch"; +import { useVoiceEarlyDeliveryPreference } from "../lib/voiceEarlyDeliveryPreference"; import { SettingsRow } from "@/shared/ui/settings-row"; import { Select, @@ -161,6 +163,7 @@ export function VoiceSettings() { setOpenAiVoice(openAiStatus.speechVoice); } }, [openAiStatus]); + const earlyDelivery = useVoiceEarlyDeliveryPreference(); const interruption = useVoiceInterruptionPreference(); const statusSounds = useStatusSoundPreference(); const siriSetup = useSiriVoiceSetup(output.backend === "siri"); @@ -234,6 +237,8 @@ export function VoiceSettings() { output.setBackend(getDefaultVoiceOutputBackend()); interruption.setMode(getDefaultVoiceInterruptionPreference().mode); statusSounds.update(getDefaultStatusSoundPreference()); + if (!earlyDelivery.setEnabled(true)) + throw new Error(t("voice.earlyDeliverySaveFailed")); setResetDialogOpen(false); } finally { setResetting(false); @@ -591,6 +596,28 @@ export function VoiceSettings() { )} /> +
+ ( + { + setResetError( + earlyDelivery.setEnabled(enabled) + ? null + : t("voice.earlyDeliverySaveFailed"), + ); + }} + /> + )} + /> +
{resetError ? (

{resetError} diff --git a/src/shared/api/__tests__/acpBackends.test.ts b/src/shared/api/__tests__/acpBackends.test.ts index c08de52a5..32c5c4f7a 100644 --- a/src/shared/api/__tests__/acpBackends.test.ts +++ b/src/shared/api/__tests__/acpBackends.test.ts @@ -924,6 +924,31 @@ describe("inbound session id translation", () => { stop(); }); + it("fences private notifications and permissions before the normal handler", async () => { + const conn = await importConnection(); + const received = vi.fn(); + const permissions = vi.fn().mockResolvedValue({ + outcome: { outcome: "selected", optionId: "approve" }, + }); + conn.setNotificationHandler({ handleSessionNotification: received }); + conn.setPermissionHandler(permissions); + await conn.getBackendConnection(LOCAL_BACKEND_ID).getClient(); + const callbacks = await latestCallbacks(); + conn.hidePrivateSessionNotifications("private-voice"); + await callbacks.sessionUpdate?.(sessionUpdatePayload("private-voice")); + expect(received).not.toHaveBeenCalled(); + await expect( + callbacks.requestPermission({ + sessionId: "private-voice", + toolCall: { toolCallId: "write", title: "write_file" }, + options: [{ optionId: "approve", name: "Allow", kind: "allow_once" }], + }), + ).resolves.toEqual({ outcome: { outcome: "cancelled" } }); + expect(permissions).not.toHaveBeenCalled(); + await callbacks.sessionUpdate?.(sessionUpdatePayload("ordinary")); + expect(received).toHaveBeenCalledOnce(); + }); + it("rewrites remote notification session ids to composite ids", async () => { const conn = await importConnection(); const received: SessionNotification[] = []; diff --git a/src/shared/api/acpApi.ts b/src/shared/api/acpApi.ts index 521c90dee..8bf6b4f6f 100644 --- a/src/shared/api/acpApi.ts +++ b/src/shared/api/acpApi.ts @@ -515,6 +515,18 @@ export async function renameSession( }); } +/** Raw ACP mode change. Callers must await success before private inference. */ +export async function setSessionMode( + sessionId: string, + modeId: string, +): Promise { + const client = await getClientForSession(sessionId); + await client.setSessionMode({ + sessionId: getWireSessionId(sessionId), + modeId, + }); +} + export async function cancelSession(sessionId: string): Promise { const client = await getClientForSession(sessionId); await client.cancel({ sessionId: getWireSessionId(sessionId) }); diff --git a/src/shared/api/acpConnection.ts b/src/shared/api/acpConnection.ts index 9f172cdc9..ee7dd0911 100644 --- a/src/shared/api/acpConnection.ts +++ b/src/shared/api/acpConnection.ts @@ -26,6 +26,13 @@ import { import { createWebSocketStream } from "./createWebSocketStream"; import { perfLog } from "@/shared/lib/perfLog"; +const privateSessionIds = new Set(); + +/** Keep a text-free fence even after cancellation: transports may emit late chunks. */ +export function hidePrivateSessionNotifications(sessionId: string): void { + privateSessionIds.add(sessionId); +} + let notificationHandler: AcpNotificationHandler | null = null; const sessionNotificationInterceptors = new Set(); @@ -101,6 +108,7 @@ function createClientCallbacks( }; if (!isCurrent()) return cancelled; const request = toRendererSessionId(args); + if (privateSessionIds.has(request.sessionId)) return cancelled; if (permissionHandler) { // Cancel immediately on detachment even if a handler never settles. // The signal also retires this generation's pending confirmation UI. @@ -142,6 +150,7 @@ function createClientCallbacks( return; } } + if (privateSessionIds.has(notification.sessionId)) return; if (notificationHandler) { await notificationHandler.handleSessionNotification(notification); } diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 6d532189a..350a60a30 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1075,6 +1075,9 @@ "useVoice": "Use {{voice}}", "voice": "Voice", "voiceLabel": "Pocket TTS voice", + "earlyDeliverySaveFailed": "Could not save early delivery setting.", + "earlyDelivery": "Early delivery (experimental)", + "earlyDeliveryDescription": "Send pending transcripts to the agent while you speak, so it can start preparing a response. Only your final transcript can trigger actions.", "statusSounds": "Status sounds", "statusSoundModes": { "off": "Off", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index f1c10728a..c4305fb1c 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1074,6 +1074,9 @@ "useVoice": "Usar {{voice}}", "voice": "Voz", "voiceLabel": "Voz de Pocket TTS", + "earlyDeliverySaveFailed": "No se pudo guardar la configuración de entrega anticipada.", + "earlyDelivery": "Entrega anticipada (experimental)", + "earlyDeliveryDescription": "Envía transcripciones pendientes al agente mientras hablas para que pueda empezar a preparar una respuesta. Solo la transcripción final puede activar acciones.", "statusSounds": "Sonidos de estado", "statusSoundModes": { "off": "Desactivados", diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index 8f61e93e9..7965ece87 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -24,6 +24,8 @@ export function buildInitScript(options?: { realAcpUrl?: string; homeDir?: string; modelId?: string; + holdAcpPrompts?: boolean; + privateVoiceDraftScenario?: "complete" | "stalled" | "mode-rejected"; personas?: unknown[]; skills?: unknown[]; projects?: unknown[]; @@ -120,6 +122,10 @@ export function buildInitScript(options?: { return ` (() => { + const PRIVATE_VOICE_DRAFT = ${JSON.stringify(options?.privateVoiceDraftScenario ?? null)}; + const HOLD_ACP_PROMPTS = ${options?.holdAcpPrompts ?? false}; + const ACP_REQUESTS = []; + const PENDING_ACP_PROMPTS = new Map(); const PERSONAS = ${personas}; const SKILLS = ${skills}; const PROJECTS = ${projects}; @@ -175,6 +181,7 @@ export function buildInitScript(options?: { lastMessageAt: session.lastMessageAt, workingDir: session.workingDir, replayText: session.replayText, + cwd: session.workingDir ?? "/tmp/alpha", providerId: session.providerId ?? "goose", modelId: session.modelId ?? null, })); @@ -211,7 +218,7 @@ export function buildInitScript(options?: { let nextEventId = 1; localStorage.setItem("goose:defaultProvider", "goose"); - if (ENABLED_EXPERIMENTS.length > 0) { + if (ENABLED_EXPERIMENTS.length > 0 && !localStorage.getItem("goose:experimental-features")) { localStorage.setItem( "goose:experimental-features", JSON.stringify({ @@ -313,8 +320,23 @@ export function buildInitScript(options?: { })(); const SKILL_SOURCES = SKILLS.map(skillToSourceEntry); const POCKET_VOICE_SPOKEN_TEXTS = []; + const VOICE_STREAM_TEXTS = new Map(); + let voiceSpeechId = 0; window.__GOOSE_E2E__ = { + acpRequests: () => clone(ACP_REQUESTS), + completeAcpPrompts: () => { + for (const [id, { socket, sessionId }] of PENDING_ACP_PROMPTS) { + socket.dispatchEvent(new MessageEvent("message", { + data: JSON.stringify(jsonRpcResult(id, { stopReason: "end_turn" })), + })); + window.__GOOSE_E2E__.emitAcpNotification({ + sessionId, + update: { sessionUpdate: "session_info_update", _meta: { goose: { activeRunId: null } } }, + }); + } + PENDING_ACP_PROMPTS.clear(); + }, listAgentSources: () => clone(AGENT_SOURCES), clearAgentSources: () => { AGENT_SOURCES = PERSONAS.map(personaToSourceEntry); @@ -336,6 +358,7 @@ export function buildInitScript(options?: { }, emitTauriEvent, pocketVoiceSpokenTexts: () => clone(POCKET_VOICE_SPOKEN_TEXTS), + pocketVoiceStreamIds: () => [...VOICE_STREAM_TEXTS.keys()], }; function nowIso() { @@ -403,21 +426,27 @@ export function buildInitScript(options?: { }); case "session/list": return jsonRpcResult(message.id, { - sessions: ACP_SESSIONS.map((session) => ({ + sessions: ACP_SESSIONS.filter((session) => !session.hidden).map((session) => ({ sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, + cwd: session.cwd, _meta: { messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, + providerId: session.providerId, + modelId: session.modelId, }, - cwd: session.workingDir, })), }); + case "_goose/unstable/session/info": { + const session = findSession(message.params?.sessionId); + return jsonRpcResult(message.id, { session: { sessionId: session.sessionId, cwd: session.cwd, _meta: { providerId: session.providerId, modelId: session.modelId } } }); + } case "session/new": { - const providerId = message.params?.meta?.provider ?? "goose"; + const providerId = message.params?._meta?.provider ?? "goose"; const sessionId = "session-" + Math.random().toString(36).slice(2, 10); - ACP_SESSIONS.unshift(buildSession(sessionId, providerId)); + ACP_SESSIONS.unshift({ ...buildSession(sessionId, providerId), hidden: message.params?._meta?.hidden === true, cwd: message.params?.cwd }); return jsonRpcResult(message.id, { sessionId }); } case "session/fork": { @@ -431,6 +460,7 @@ export function buildInitScript(options?: { providerId: source?.providerId ?? "goose", modelId: source?.modelId ?? null, conversationBefore: message.params?._meta?.conversationBefore, + mode: null, }; ACP_SESSIONS.unshift(copy); return jsonRpcResult(message.id, { @@ -460,7 +490,10 @@ export function buildInitScript(options?: { update: { sessionUpdate: "user_message_chunk", content: { type: "text", text: session.replayText } }, }); } - return jsonRpcResult(message.id, {}); + return jsonRpcResult(message.id, { configOptions: [ + { id: "provider", kind: { type: "select", currentValue: session?.providerId ?? "openai", options: [] } }, + { id: "model", category: "model", kind: { type: "select", currentValue: session?.modelId ?? "gpt-4.1", options: [] } }, + ] }); } case "session/set_config_option": { const session = findSession(message.params?.sessionId); @@ -476,6 +509,22 @@ export function buildInitScript(options?: { } return jsonRpcResult(message.id, {}); } + case "session/set_mode": { + const session = findSession(message.params?.sessionId); + if (PRIVATE_VOICE_DRAFT === "mode-rejected") return { jsonrpc: "2.0", id: message.id, error: { code: -32601, message: "Chat mode unavailable" } }; + if (session) session.mode = message.params.modeId; + return jsonRpcResult(message.id, {}); + } + case "session/delete": { + const index = ACP_SESSIONS.findIndex((session) => session.sessionId === message.params?.sessionId); + if (index >= 0) ACP_SESSIONS.splice(index, 1); + return jsonRpcResult(message.id, {}); + } + case "_goose/unstable/session/steer": + return jsonRpcResult(message.id, { + runId: message.params.expectedRunId, + messageId: "mock-steer-" + message.id, + }); case "session/prompt": { const session = findSession(message.params?.sessionId); if (session) { @@ -530,6 +579,10 @@ export function buildInitScript(options?: { isConfigured: entry.configured, })), }); + case "_goose/unstable/providers/secrets/list": + return jsonRpcResult(message.id, { secrets: [{ provider: "openai", name: "OPENAI_API_KEY", storage: "secret_store", status: "unknown", configured: true, hasSecret: true, canDelete: true, canConfigure: false }] }); + case "_goose/unstable/defaults/read": + case "_goose/unstable/defaults/save": case "_goose/defaults/read": case "_goose/defaults/save": return jsonRpcResult(message.id, { @@ -673,6 +726,24 @@ export function buildInitScript(options?: { send(raw) { const message = JSON.parse(raw); + ACP_REQUESTS.push(clone(message)); + // Semantic model boundary: private inference is only accepted after + // a successful chat-mode transition, not merely an audience hint. + if (PRIVATE_VOICE_DRAFT && message.method === "session/prompt" && findSession(message.params.sessionId)?.hidden) { + const session = findSession(message.params.sessionId); + if (session?.mode !== "chat" || !session.providerId || !session.modelId) throw new Error("Private model invoked without chat mode and configured model"); + if (PRIVATE_VOICE_DRAFT === "complete") { + queueMicrotask(() => { + window.__GOOSE_E2E__.emitAcpNotification({ sessionId: session.sessionId, update: { sessionUpdate: "agent_message_chunk", content: { type: "text", text: "PRIVATE DRAFT: I would set the temperature to 10 degrees." } } }); + this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(jsonRpcResult(message.id, { stopReason: "end_turn" })) })); + }); + return; + } + } + if (HOLD_ACP_PROMPTS && message.method === "session/prompt") { + PENDING_ACP_PROMPTS.set(message.id, { socket: this, sessionId: message.params.sessionId }); + return; + } const response = message && typeof message === "object" && "id" in message ? handleAcpRequest(message) @@ -802,6 +873,32 @@ export function buildInitScript(options?: { ); emitTauriEvent("openai-voice:settings-changed", null); return Promise.resolve(null); + case "prepare_native_voice_assistant_speech": + return Promise.resolve({ outcome: "admitted", speechId: ++voiceSpeechId }); + case "start_pocket_voice_stream": + case "start_siri_voice_stream": + VOICE_STREAM_TEXTS.set(args.streamId, ""); + return Promise.resolve(true); + case "append_pocket_voice_stream": + case "append_siri_voice_stream": { + VOICE_STREAM_TEXTS.set(args.streamId, (VOICE_STREAM_TEXTS.get(args.streamId) ?? "") + args.text); + return Promise.resolve(null); + } + case "flush_pocket_voice_stream": + case "flush_siri_voice_stream": + case "finish_pocket_voice_stream": + case "finish_siri_voice_stream": { + const text = VOICE_STREAM_TEXTS.get(args.streamId); + if (text) { + POCKET_VOICE_SPOKEN_TEXTS.push(text); + VOICE_STREAM_TEXTS.set(args.streamId, ""); + } + if (cmd.startsWith("finish")) queueMicrotask(() => emitTauriEvent( + cmd.includes("siri") ? "siri-voice:stream-event" : "pocket-voice:stream-event", + { streamId: args.streamId, state: "completed", error: null }, + )); + return Promise.resolve(null); + } case "speak_pocket_voice": POCKET_VOICE_SPOKEN_TEXTS.push(args?.text); return new Promise((resolve) => @@ -876,6 +973,8 @@ export function buildInitScript(options?: { }); // ---- Sessions / Misc ---- + case "list_berd_app_skills": + return Promise.resolve({ skills: [] }); case "get_session_window_support": return Promise.resolve({ supported: false, @@ -891,12 +990,15 @@ export function buildInitScript(options?: { return Promise.resolve(null); case "list_sessions": return Promise.resolve( - ACP_SESSIONS.map((session) => ({ + ACP_SESSIONS.filter((session) => !session.hidden).map((session) => ({ sessionId: session.sessionId, title: session.title, updatedAt: session.updatedAt, + cwd: session.cwd, messageCount: session.messageCount, lastMessageAt: session.lastMessageAt, + providerId: session.providerId, + modelId: session.modelId, conversationBefore: session.conversationBefore, })), ); diff --git a/tests/e2e/voice-conversation.spec.ts b/tests/e2e/voice-conversation.spec.ts index 264aa0f74..b241f845c 100644 --- a/tests/e2e/voice-conversation.spec.ts +++ b/tests/e2e/voice-conversation.spec.ts @@ -2,6 +2,8 @@ import { test as base, expect, type Page } from "@playwright/test"; import { buildInitScript } from "./fixtures/tauri-mock"; +base.use({ screenshot: "off", video: "off", trace: "off" }); + const SESSION_ID = "voice-session"; const ASSISTANT_MESSAGE_ID = "assistant-message"; const SPOKEN_TEXT = "This playback-confirmed reply should be visible."; @@ -34,7 +36,7 @@ const test = base.extend<{ voice: Page }>({ test("queues assistant text through native playback", async ({ voice: page, -}, testInfo) => { +}) => { await page.goto("/"); await page.getByText("Voice playback", { exact: true }).click(); @@ -51,7 +53,7 @@ test("queues assistant text through native playback", async ({ harness.emitTauriEvent("voice-conversation:event", { type: "startup", sessionId, - ownerWindowLabel: "voice-e2e-owner", + ownerWindowLabel: "main", line: "Voice conversation ready", revision: 2, }); @@ -111,8 +113,507 @@ test("queues assistant text through native playback", async ({ ), ) .toEqual([SPOKEN_TEXT]); - await testInfo.attach("playback-confirmed-text-status", { - body: await page.screenshot(), - contentType: "image/png", - }); }); + +// These exercise the rendered controller and ACP adapter, not native recognition +// or a real model. Prompt completion is held so pause/final races are deterministic. +type AcpRequest = { + method: string; + params?: { + sessionId?: string; + modeId?: string; + prompt?: { text?: string; annotations?: { audience?: string[] } }[]; + _meta?: { goose?: { voiceUtteranceId?: string } }; + }; +}; +type VoiceHarness = { + emitTauriEvent: (event: string, payload: unknown) => void; + emitAcpNotification: (params: unknown) => void; + acpRequests: () => AcpRequest[]; + completeAcpPrompts: () => void; + pocketVoiceSpokenTexts: () => string[]; + pocketVoiceStreamIds: () => string[]; +}; + +async function nativeEvent(page: Page, payload: object) { + await page.evaluate( + (payload) => { + const harness = ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__; + harness.emitTauriEvent("voice-conversation:event", payload); + }, + { sessionId: SESSION_ID, revision: 2, ...payload }, + ); +} + +async function requests(page: Page) { + return page.evaluate(() => + (window as typeof window & { __GOOSE_E2E__: VoiceHarness }).__GOOSE_E2E__ + .acpRequests() + .filter((request) => + ["session/prompt", "_goose/unstable/session/steer"].includes( + request.method, + ), + ), + ); +} + +async function spoken(page: Page) { + return page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.pocketVoiceSpokenTexts(), + ); +} + +async function completePrompts(page: Page) { + await page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.completeAcpPrompts(), + ); +} + +async function assistantOutput(page: Page, text: string, runId: string) { + await page.evaluate( + ({ text, runId, sessionId }) => { + const harness = ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__; + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "session_info_update", + _meta: { goose: { activeRunId: runId } }, + }, + }); + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + messageId: `message-${runId}`, + content: { type: "text", text }, + }, + }); + // A tool boundary flushes the speech sentence without completing the run. + harness.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "tool_call", + toolCallId: `tool-${runId}`, + title: "read_file", + rawInput: { path: "/tmp/voice-e2e.txt" }, + }, + }); + }, + { text, runId, sessionId: SESSION_ID }, + ); +} + +const utterance = { + lifecycleId: "early-delivery-lifecycle", + id: "utterance-1", + text: "Set the temperature to 10 degrees.", +}; + +for (const scenario of ["complete", "stalled", "mode-rejected"] as const) { + base.describe(`isolated early voice (${scenario})`, () => { + base.beforeEach(async ({ page }) => { + await page.addInitScript({ + content: buildInitScript({ + holdAcpPrompts: true, + privateVoiceDraftScenario: scenario, + sessions: [ + { + sessionId: SESSION_ID, + title: "Early voice", + messageCount: 1, + providerId: "openai", + modelId: "gpt-4.1", + workingDir: "/workspace", + }, + ], + voiceConversationStatus: { + available: true, + unavailableReason: null, + lifecycle: "stopped", + sessionId: null, + ownerWindowLabel: null, + revision: 1, + }, + enabledExperiments: ["voice-conversation"], + }), + }); + await page.goto("/"); + await page.getByText("Early voice", { exact: true }).click(); + await expect(page.getByRole("textbox").first()).toBeVisible(); + await page.evaluate( + () => + new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ), + ); + await nativeEvent(page, { + type: "startup", + ownerWindowLabel: "main", + line: "Ready", + }); + }); + + if (scenario === "complete") + base( + "early delivery toggle persists and gates only private preparation", + async ({ page }) => { + const openSettings = async () => { + await page.locator("[data-sidebar-nav-id=settings]").click(); + await page.locator("[data-sidebar-nav-id=settings-voice]").click(); + }; + const openChat = async () => { + await page + .getByTestId("sidebar-primary-nav-panel") + .getByRole("button", { name: "Back", exact: true }) + .click(); + await page + .getByRole("button", { name: "Early voice", exact: true }) + .click(); + await expect(page.getByRole("textbox").first()).toBeVisible(); + }; + const toggle = page.getByRole("switch", { + name: "Early delivery (experimental)", + exact: true, + }); + const allRequests = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.acpRequests(), + ); + const hiddenSessions = async () => + (await allRequests()).filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); + + await openSettings(); + await expect(toggle).toBeChecked(); + await toggle.uncheck(); + await expect(toggle).not.toBeChecked(); + await openChat(); + await page.goto("/"); + await page.reload(); + await openSettings(); + await expect(toggle).not.toBeChecked(); + await openChat(); + await nativeEvent(page, { + type: "startup", + ownerWindowLabel: "main", + line: "Ready", + }); + await nativeEvent(page, { type: "interimUser", ...utterance }); + // Drain renderer work before asserting that the disabled interim was ignored. + await page.evaluate( + () => + new Promise((resolve) => + requestAnimationFrame(() => + requestAnimationFrame(() => resolve()), + ), + ), + ); + expect(await hiddenSessions()).toHaveLength(0); + expect(await requests(page)).toHaveLength(0); + await nativeEvent(page, { + type: "user", + ...utterance, + deliveryAttempts: 0, + }); + await expect.poll(() => requests(page)).toHaveLength(1); + const final = (await requests(page))[0]; + expect(final.method).toBe("session/prompt"); + expect(final.params?.sessionId).toBe(SESSION_ID); + expect(final.params?.prompt).toContainEqual({ + type: "text", + text: utterance.text, + }); + expect(await hiddenSessions()).toHaveLength(0); + await expect( + page.getByText(utterance.text, { exact: true }), + ).toHaveCount(1); + await completePrompts(page); + + await openSettings(); + await toggle.check(); + await expect(toggle).toBeChecked(); + await openChat(); + await nativeEvent(page, { + type: "interimUser", + ...utterance, + id: "utterance-reenabled", + }); + await expect.poll(hiddenSessions).toHaveLength(1); + await expect + .poll(async () => + (await allRequests()).filter( + (r) => r.method === "session/set_mode", + ), + ) + .toHaveLength(1); + const mode = (await allRequests()).find( + (r) => r.method === "session/set_mode", + ); + expect(mode?.params?.sessionId).not.toBe(SESSION_ID); + expect(mode?.params?.modeId).toBe("chat"); + const creationWire = await allRequests(); + expect( + creationWire.filter((r) => r.method === "session/fork"), + ).toHaveLength(0); + const created = creationWire.filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); + expect(created).toHaveLength(1); + expect(created[0].params?.cwd).toBe("/workspace"); + expect(created[0].params?._meta).toMatchObject({ + hidden: true, + provider: "openai", + }); + expect(creationWire).toContainEqual( + expect.objectContaining({ + method: "session/set_config_option", + params: expect.objectContaining({ + sessionId: mode?.params?.sessionId, + configId: "model", + value: "gpt-4.1", + }), + }), + ); + + await expect + .poll(async () => + (await requests(page)).filter( + (r) => r.params?.sessionId === mode?.params?.sessionId, + ), + ) + .toHaveLength(1); + expect( + (await requests(page)).filter( + (r) => r.params?.sessionId === SESSION_ID, + ), + ).toHaveLength(1); + await nativeEvent(page, { + type: "recognitionPending", + pending: false, + }); + }, + ); + + for (const busy of [false, true]) + base( + `final has sole tool authority, bounded delivery and later utterance (busy: ${busy})`, + async ({ page }) => { + if (busy) + await assistantOutput( + page, + "An unrelated running answer.", + "unrelated-run", + ); + await nativeEvent(page, { + type: "activity", + activity: "user-speaking", + }); + await nativeEvent(page, { type: "interimUser", ...utterance }); + const allRequests = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.acpRequests(), + ); + await expect + .poll(async () => + (await allRequests()).filter( + (r) => r.method === "session/set_mode", + ), + ) + .toHaveLength(1); + const mode = (await allRequests()).find( + (r) => r.method === "session/set_mode", + ); + const privateId = mode?.params?.sessionId; + if (!privateId) throw new Error("Missing hidden private session"); + expect(privateId).not.toBe(SESSION_ID); + expect(mode?.params?.modeId).toBe("chat"); + const creationWire = await allRequests(); + expect( + creationWire.filter((r) => r.method === "session/fork"), + ).toHaveLength(0); + const created = creationWire.filter( + (r) => + r.method === "session/new" && r.params?._meta?.hidden === true, + ); + expect(created).toHaveLength(1); + expect(created[0].params?.cwd).toBe("/workspace"); + expect(created[0].params?._meta).toMatchObject({ + hidden: true, + provider: "openai", + }); + expect(creationWire).toContainEqual( + expect.objectContaining({ + method: "session/set_config_option", + params: expect.objectContaining({ + sessionId: mode?.params?.sessionId, + configId: "model", + value: "gpt-4.1", + }), + }), + ); + + if (scenario !== "mode-rejected") { + await expect.poll(() => requests(page)).toHaveLength(1); + expect((await requests(page))[0].params?.sessionId).toBe(privateId); + const wire = await allRequests(); + expect( + wire.findIndex((r) => r.method === "session/set_mode"), + ).toBeLessThan( + wire.findIndex((r) => r.method === "session/prompt"), + ); + } else expect(await requests(page)).toHaveLength(0); + await expect( + page.getByText(utterance.text, { exact: true }), + ).toHaveCount(0); + await expect( + page.getByText("PRIVATE DRAFT:", { exact: false }), + ).toHaveCount(0); + const finalText = busy + ? utterance.text + : "Set the temperature to -10 degrees."; + // The private draft has started while speaking. The final arrives + // before VAD idle, and remains the sole authoritative user turn. + const started = Date.now(); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + const mainRequests = async () => + (await requests(page)).filter( + (r) => r.params?.sessionId === SESSION_ID, + ); + await expect + .poll(mainRequests, { timeout: 1800, intervals: [20, 50] }) + .toHaveLength(1); + expect(Date.now() - started).toBeLessThan(1800); + await nativeEvent(page, { + type: "activity", + activity: "user-idle", + }); + const final = (await mainRequests())[0]; + expect(final.method).toBe( + busy ? "_goose/unstable/session/steer" : "session/prompt", + ); + expect( + final.params?.prompt?.find((b) => b.text === finalText) + ?.annotations, + ).toBeUndefined(); + const context = + final.params?.prompt + ?.filter((b) => b.annotations?.audience?.includes("assistant")) + .map((b) => b.text) + .join("\n") ?? ""; + if (scenario === "complete") { + expect(context).toContain("PRIVATE DRAFT"); + expect(context).toContain("perform any requested actions normally"); + } else expect(context).not.toContain("PRIVATE DRAFT"); + expect(context).not.toContain("do not repeat"); + await expect(page.getByText(finalText, { exact: true })).toHaveCount( + 1, + ); + await nativeEvent(page, { + type: "user", + ...utterance, + text: finalText, + deliveryAttempts: 0, + }); + expect(await mainRequests()).toHaveLength(1); + const cancellations = (await allRequests()).filter( + (r) => r.method === "session/cancel", + ); + if (scenario !== "mode-rejected") + expect(cancellations.length).toBeGreaterThan(0); + expect( + cancellations.every((r) => r.params?.sessionId === privateId), + ).toBe(true); + // Late private output is fenced even after bounded cancellation/cleanup. + await page.evaluate( + (sessionId) => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__.emitAcpNotification({ + sessionId, + update: { + sessionUpdate: "agent_message_chunk", + content: { type: "text", text: "LATE PRIVATE OUTPUT" }, + }, + }), + privateId, + ); + await expect( + page.getByText("LATE PRIVATE OUTPUT", { exact: false }), + ).toHaveCount(0); + expect( + (await spoken(page)).every((text) => !text.includes("PRIVATE")), + ).toBe(true); + if (!busy) + await assistantOutput( + page, + "The final action has been performed.", + "final-run", + ); + await nativeEvent(page, { + type: "user", + ...utterance, + id: "utterance-2", + text: "Now report the result.", + deliveryAttempts: 0, + }); + await expect.poll(mainRequests).toHaveLength(2); + expect((await mainRequests())[1].method).toBe( + "_goose/unstable/session/steer", + ); + await completePrompts(page); + }, + ); + + base( + "recognition ending without final permits another private utterance", + async ({ page }) => { + await nativeEvent(page, { type: "interimUser", ...utterance }); + const hiddenSessions = () => + page.evaluate(() => + ( + window as typeof window & { __GOOSE_E2E__: VoiceHarness } + ).__GOOSE_E2E__ + .acpRequests() + .filter( + (r) => + r.method === "session/new" && + r.params?._meta?.hidden === true, + ), + ); + await expect.poll(hiddenSessions).toHaveLength(1); + await nativeEvent(page, { type: "recognitionPending", pending: false }); + await nativeEvent(page, { + type: "interimUser", + ...utterance, + id: "utterance-2", + }); + await expect.poll(hiddenSessions).toHaveLength(2); + expect( + (await requests(page)).every( + (r) => r.params?.sessionId !== SESSION_ID, + ), + ).toBe(true); + }, + ); + }); +}