diff --git a/README.md b/README.md index c5519c8506c6..c1eb9e04db30 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Your existing subscriptions are used directly — MT Code sells nothing and adds - **Resume on restart** — threads and agents that were running when the app closed automatically pick up where they left off at the next launch. - **Cross-thread references** — type `#` in the composer to reference another thread; the agent can read that thread's transcript. - **Thread-to-thread messaging** — agents can list sibling threads and send messages between them, so parallel work can coordinate. +- **Agent-chosen computers** — agents can send a task to another machine already connected in MT Code (this computer, SSH, T3 Connect, or a paired backend) without you changing **Run on**. - **Better "Open in editor" on macOS** — detects Cursor, VS Code Insiders, VSCodium, Trae, Kiro, and JetBrains IDEs by their app bundles, even when their CLI shims aren't installed. - **Installs alongside official T3 Code** — its own bundle ID (`com.munim.t3code`) and its own isolated data directory, so it never touches the official app's settings or sessions. @@ -48,6 +49,7 @@ Full docs live in [docs/](./docs): - [Desktop notifications](./docs/user/desktop-notifications.md) - [Remote access from a phone or another machine](./docs/user/remote-access.md) - [Thread messaging](./docs/user/thread-messaging.md) +- [Sending work to another computer](./docs/user/computer-routing.md) - [Keeping app and server in sync](./docs/user/updating.md) - [Source control integrations](./docs/user/source-control.md) - Multiple accounts: [Codex](./docs/user/providers-codex.md) · [Claude](./docs/user/providers-claude.md) diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 9a51725478e1..6c80e8cd0011 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -235,6 +235,13 @@ const config: ExpoConfig = { }, plugins: [ "expo-asset", + [ + "expo-audio", + { + microphonePermission: "Allow T3 Code to access your microphone for voice dictation.", + enableBackgroundRecording: false, + }, + ], [ "expo-font", { diff --git a/apps/mobile/package.json b/apps/mobile/package.json index de53a37c995b..6711f740e92c 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -73,6 +73,7 @@ "effect": "catalog:", "expo": "~56.0.12", "expo-asset": "~56.0.17", + "expo-audio": "~56.0.13", "expo-auth-session": "~56.0.14", "expo-blur": "~56.0.3", "expo-build-properties": "~56.0.19", diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 8b219afcc078..85df586cc183 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -9,6 +9,7 @@ import { SafeAreaProvider } from "react-native-safe-area-context"; import { createStaticNavigation } from "@react-navigation/native"; import { RegistryContext } from "@effect/atom-react"; +import { ComputerTaskHosts } from "./components/ComputerTaskHosts"; import { ConfirmDialogHost } from "./components/ConfirmDialogHost"; import { CloudAuthProvider } from "./features/cloud/CloudAuthProvider"; import { prepareNativeShowcaseCapture } from "./features/showcase/nativeShowcaseScene"; @@ -97,6 +98,7 @@ function AppContent() { + {/* Anchored-menu overlays render here — in-window, so the keyboard stays up while a dropdown is open. */} diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index 38101a993e64..486f480c887f 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -57,6 +57,7 @@ import { SettingsEnvironmentsRouteScreen } from "./features/settings/SettingsEnv import { SettingsEnvironmentRenameRouteScreen } from "./features/settings/SettingsEnvironmentRenameRouteScreen"; import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen"; import { SettingsProjectGroupingRouteScreen } from "./features/settings/SettingsProjectGroupingRouteScreen"; +import { SettingsVoiceDictationRouteScreen } from "./features/settings/SettingsVoiceDictationRouteScreen"; import { UsageRouteScreen } from "./features/usage/UsageRouteScreen"; import { SettingsRouteScreen } from "./features/settings/SettingsRouteScreen"; import { ShowcaseCaptureCoordinator } from "./features/showcase/ShowcaseCaptureCoordinator"; @@ -185,6 +186,13 @@ const SettingsContentStack = createNativeStackNavigator({ title: "Project Grouping", }, }), + SettingsVoiceDictation: createNativeStackScreen({ + screen: SettingsVoiceDictationRouteScreen, + linking: "voice-dictation", + options: { + title: "Voice Dictation", + }, + }), SettingsClientStorage: createNativeStackScreen({ screen: SettingsClientStorageRouteScreen, linking: "client-storage", diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 74308467a052..199431d68602 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -48,6 +48,7 @@ import { IconLetterSpacing, IconLink, IconMessage, + IconMicrophone, IconMinus, IconMoon, IconNetwork, @@ -126,6 +127,7 @@ const ANDROID_ICON_BY_SF_SYMBOL: Partial> = { "line.3.horizontal.decrease.circle": IconFilter, "line.3.horizontal.decrease.circle.fill": IconFilter, magnifyingglass: IconSearch, + mic: IconMicrophone, paintbrush: IconPalette, "person.crop.circle": IconUserCircle, pin: IconPin, diff --git a/apps/mobile/src/components/ComputerTaskHosts.tsx b/apps/mobile/src/components/ComputerTaskHosts.tsx new file mode 100644 index 000000000000..94e2b9dc148c --- /dev/null +++ b/apps/mobile/src/components/ComputerTaskHosts.tsx @@ -0,0 +1,244 @@ +import { RegistryContext, useAtomValue } from "@effect/atom-react"; +import { + computerPeerFromPresentation, + ComputerTaskDispatchError, + matchProject, + requireMatchedProject, +} from "@t3tools/client-runtime/computers"; +import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { + MessageId, + ThreadId, + type ComputerPeer, + type ComputerTaskHost as ComputerTaskHostPayload, + type ComputerTaskSendRequest, + type ComputerTaskSendResult, + type ComputerTaskStreamEvent, + type EnvironmentId, + type OrchestrationShellSnapshot, +} from "@t3tools/contracts"; +import { AsyncResult, Atom, AtomRegistry } from "effect/unstable/reactivity"; +import { useCallback, useContext, useEffect, useMemo, useState } from "react"; + +import { environmentCatalog } from "../connection/catalog"; +import { computerEnvironment } from "../state/computers"; +import { useEnvironments } from "../state/environments"; +import { environmentPresentations } from "../state/presentation"; +import { environmentSnapshotAtom } from "../state/shell"; +import { threadEnvironment } from "../state/threads"; +import { useAtomCommand } from "../state/use-atom-command"; + +const TARGET_READY_TIMEOUT_MS = 20_000; +const TARGET_READY_POLL_MS = 200; + +function createComputerTaskClientId(): string { + const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16)); + return `computers-${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; +} + +function randomId(): string { + const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16)); + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +type RequestStreamResult = AsyncResult.AsyncResult; + +function createComputerTaskRequestConsumerAtom(options: { + readonly requestsAtom: Atom.Atom>; + readonly clientId: ComputerTaskHostPayload["clientId"]; + readonly handle: (request: ComputerTaskSendRequest) => Promise; + readonly respond: (response: { + readonly clientId: ComputerTaskHostPayload["clientId"]; + readonly connectionId: string; + readonly requestId: string; + readonly ok: boolean; + readonly result?: ComputerTaskSendResult; + readonly error?: { + readonly code: + | "dispatch_failed" + | "project_not_found" + | "project_ambiguous" + | "computer_offline"; + readonly detail: string; + }; + }) => Promise; +}): Atom.Atom { + return Atom.make((get) => { + const consume = (result: RequestStreamResult) => { + if (!AsyncResult.isSuccess(result)) return; + const event = result.value; + if (event.type === "connected") return; + void options.handle(event.request).then( + (value) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: true, + result: value, + }), + (error) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: false, + error: { + code: error instanceof ComputerTaskDispatchError ? error.code : "dispatch_failed", + detail: + error instanceof Error + ? error.message + : "Could not start the task on that computer.", + }, + }), + ); + }; + const initial = get.once(options.requestsAtom); + get.subscribe(options.requestsAtom, consume); + queueMicrotask(() => consume(initial)); + }).pipe(Atom.withLabel(`computer-task-consumer:${options.clientId}`)); +} + +export function ComputerTaskHosts() { + const { environments } = useEnvironments(); + const computers = useMemo( + () => environments.map((environment) => computerPeerFromPresentation(environment)), + [environments], + ); + if (environments.length === 0) return null; + return ( + <> + {environments.map((environment) => ( + + ))} + + ); +} + +function ComputerTaskHost(props: { + readonly environmentId: EnvironmentId; + readonly computers: ReadonlyArray; +}) { + const { environmentId, computers } = props; + const registry = useContext(RegistryContext); + const [clientId] = useState(createComputerTaskClientId); + const [connectHost] = useState(() => ({ clientId, computers })); + const requestsAtom = computerEnvironment.requests({ environmentId, input: connectHost }); + const respond = useAtomCommand(computerEnvironment.respond, { reportFailure: false }); + const sync = useAtomCommand(computerEnvironment.sync, { reportFailure: false }); + const startTurn = useAtomCommand(threadEnvironment.startTurn, { reportFailure: false }); + const retryNow = useAtomCommand(environmentCatalog.retryNow, { reportFailure: false }); + const thisPresentation = useAtomValue(environmentPresentations.presentationAtom(environmentId)); + useAtomValue(environmentSnapshotAtom(environmentId)); + + useEffect(() => { + if (thisPresentation?.connection.phase !== "connected") return; + void sync({ environmentId, input: { clientId, computers } }); + }, [clientId, computers, environmentId, sync, thisPresentation?.connection.phase]); + + const handleRequest = useCallback( + async (request: ComputerTaskSendRequest): Promise => { + const targetId = request.computer.environmentId; + const livePresentation = registry + .get(environmentPresentations.presentationsAtom) + .get(targetId); + if (livePresentation && livePresentation.connection.phase !== "connected") { + await retryNow(targetId); + } + const snapshot = await waitForTargetSnapshot(registry, targetId); + const matched = requireMatchedProject( + matchProject({ + projects: snapshot.projects, + hint: request.projectHint, + sourceTitle: request.source.projectTitle, + sourceWorkspaceRoot: request.source.projectWorkspaceRoot, + }), + ); + const threadId = ThreadId.make(randomId()); + const messageId = MessageId.make(randomId()); + const createdAt = new Date().toISOString(); + const startResult = await startTurn({ + environmentId: targetId, + input: { + threadId, + message: { + messageId, + role: "user", + text: request.message, + attachments: [], + }, + modelSelection: request.modelSelection, + titleSeed: request.title, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + bootstrap: { + createThread: { + projectId: matched.id, + title: request.title, + modelSelection: request.modelSelection, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + branch: null, + worktreePath: null, + createdAt, + }, + }, + createdAt, + }, + }); + if (startResult._tag === "Failure") { + throw squashAtomCommandFailure(startResult); + } + return { + environmentId: targetId, + threadId, + projectId: matched.id, + }; + }, + [registry, retryNow, startTurn], + ); + + const consumerAtom = useMemo( + () => + createComputerTaskRequestConsumerAtom({ + requestsAtom, + clientId, + handle: handleRequest, + respond: (response) => respond({ environmentId, input: response }), + }), + [clientId, environmentId, handleRequest, requestsAtom, respond], + ); + useAtomValue(consumerAtom); + return null; +} + +async function waitForTargetSnapshot( + registry: AtomRegistry.AtomRegistry, + targetId: EnvironmentId, +): Promise { + const deadline = Date.now() + TARGET_READY_TIMEOUT_MS; + let lastPhase: string | null = null; + while (Date.now() < deadline) { + const presentation = registry.get(environmentPresentations.presentationsAtom).get(targetId); + lastPhase = presentation?.connection.phase ?? null; + const snapshot = registry.get(environmentSnapshotAtom(targetId)); + if (lastPhase === "connected" && snapshot !== null) { + return snapshot; + } + await sleep(TARGET_READY_POLL_MS); + } + throw new ComputerTaskDispatchError( + "computer_offline", + lastPhase === "connected" + ? "That computer connected but T3 has not loaded its projects yet." + : "Could not reach that computer. Keep T3 Code running there, or pick it in Run on first.", + ); +} diff --git a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx index 8ce67e3a6b75..ae784c6056a9 100644 --- a/apps/mobile/src/features/settings/SettingsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsRouteScreen.tsx @@ -47,6 +47,10 @@ import { SettingsRow } from "./components/SettingsRow"; import { SettingsSection } from "./components/SettingsSection"; import { SettingsSwitchRow } from "./components/SettingsSwitchRow"; import { resolveAgentAwarenessPlatformPresentation } from "./SettingsRouteScreen.logic"; +import { + activeMobileVoiceTranscriptionConfig, + useMobileVoiceTranscriptionSettings, +} from "../voice-dictation/voiceTranscriptionSettings"; type NotificationStatus = "checking" | "enabled" | "disabled" | "unsupported"; type LiveActivityStatus = "checking" | "enabled" | "disabled" | "signed-out" | "linking"; @@ -535,10 +539,22 @@ function GeneralSettingsSection() { preferencesResult.value.autoSettleOnMerge !== false; const steerActiveTurns = !AsyncResult.isSuccess(preferencesResult) || preferencesResult.value.steerActiveTurns !== false; + const voiceTranscription = useMobileVoiceTranscriptionSettings(); + const voiceTranscriptionConfig = activeMobileVoiceTranscriptionConfig(voiceTranscription); return ( + 0 + ? "Enabled" + : "Set up" + } + target="SettingsVoiceDictation" + /> ("openai"); + const [draftProviders, setDraftProviders] = + useState(EMPTY_PROVIDER_SETTINGS); + const [draftInitialized, setDraftInitialized] = useState(false); + const foreground = useThemeColor("--color-foreground"); + const placeholder = useThemeColor("--color-foreground-muted"); + const checkmarkColor = useThemeColor("--color-icon"); + const providerConfig = mobileVoiceTranscriptionProviderConfig(draftProvider); + const providerSettings = draftProviders[draftProvider]; + + useEffect(() => { + if (!settings.loaded || draftInitialized) return; + setDraftProvider(settings.provider); + setDraftProviders({ + openai: { ...settings.providers.openai }, + groq: { ...settings.providers.groq }, + }); + setDraftInitialized(true); + }, [draftInitialized, settings.loaded, settings.provider, settings.providers]); + + const updateSelectedProvider = ( + patch: Partial, + ) => { + setDraftProviders((current) => ({ + ...current, + [draftProvider]: { ...current[draftProvider], ...patch }, + })); + }; + + const save = async () => { + try { + await saveMobileVoiceTranscriptionSettings({ + provider: draftProvider, + providers: draftProviders, + }); + Alert.alert( + providerSettings.apiKey.trim() ? "Voice dictation enabled" : "Voice dictation disabled", + providerSettings.apiKey.trim() + ? `The microphone now uses ${providerConfig.label} with ${providerSettings.model.trim()}.` + : `No ${providerConfig.label} API key is selected, so the microphone is hidden.`, + ); + } catch (cause) { + Alert.alert( + "Could not save voice dictation", + cause instanceof Error ? cause.message : "Try again.", + ); + } + }; + + return ( + + + + {MOBILE_VOICE_TRANSCRIPTION_PROVIDERS.map((provider, index) => ( + setDraftProvider(provider.id)} + className={ + index === 0 + ? "flex-row items-center gap-4 p-4" + : "flex-row items-center gap-4 border-t border-border-subtle p-4" + } + > + + {provider.label} + + {provider.id === "openai" + ? "GPT transcription models" + : "Fast OpenAI-compatible Whisper models"} + + + {draftProvider === provider.id ? ( + + ) : null} + + ))} + + + + + updateSelectedProvider({ apiKey })} + className="rounded-xl bg-subtle px-4 py-3 text-base" + style={{ color: foreground }} + accessibilityLabel={`${providerConfig.label} API key for voice dictation`} + /> + + + + + + + Choose a known model or enter any compatible model ID. + + updateSelectedProvider({ model })} + className="rounded-xl bg-subtle px-4 py-3 text-base" + style={{ color: foreground }} + accessibilityLabel="Voice transcription model ID" + /> + + {providerConfig.modelOptions.map((model) => ( + updateSelectedProvider({ model: model.id })} + className="flex-row items-center gap-4 border-t border-border-subtle p-4" + > + + {model.label} + {model.id} + + {providerSettings.model.trim() === model.id ? ( + + ) : null} + + ))} + + + void save()} + className="h-11 items-center justify-center rounded-full bg-primary disabled:opacity-50" + > + {settings.saving ? ( + + ) : ( + Save + )} + + + + Each provider keeps its own key and model in the iPhone secure store. Audio is sent + directly to the selected provider. Clearing the selected key hides the microphone. + + {settings.error ? ( + {settings.error} + ) : null} + + + ); +} diff --git a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts index 7189fdc2ebe5..e49bbd5fc39b 100644 --- a/apps/mobile/src/features/settings/components/settings-sheet-targets.ts +++ b/apps/mobile/src/features/settings/components/settings-sheet-targets.ts @@ -3,6 +3,7 @@ export type SettingsSheetTarget = | "SettingsArchive" | "SettingsAppearance" | "SettingsProjectGrouping" + | "SettingsVoiceDictation" | "SettingsClientStorage" | "SettingsUsage"; diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index b1a7624b7dd6..b41b4a044250 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -13,6 +13,7 @@ import { serializeComposerFileLink, type ComposerTrigger, } from "@t3tools/shared/composerTrigger"; +import { appendVoiceTranscript as appendVoiceTranscriptText } from "@t3tools/shared/voiceTranscription"; import { StackActions, useFocusEffect, useNavigation } from "@react-navigation/native"; import type { ReactNode } from "react"; import { memo, useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; @@ -75,6 +76,12 @@ import { useThreadSettingsSheetPresentation, type NavigationWithFinishTransitioning, } from "./use-thread-settings-sheet-presentation"; +import { MobileVoiceTranscriptionPanel } from "../voice-dictation/MobileVoiceTranscriptionPanel"; +import { useMobileVoiceTranscription } from "../voice-dictation/useMobileVoiceTranscription"; +import { + activeMobileVoiceTranscriptionConfig, + useMobileVoiceTranscriptionSettings, +} from "../voice-dictation/voiceTranscriptionSettings"; /** * Height of the collapsed composer (pill + vertical padding, excluding safe-area inset). @@ -543,8 +550,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer // ── Handle command selection ────────────────────────────── const { onChangeDraftMessage, onUpdateInteractionMode, draftMessage, onSendMessage } = props; + const voiceTranscriptionTargetKey = scopedThreadKey(props.environmentId, props.selectedThread.id); + const voiceTranscriptionTargetKeyRef = useRef(voiceTranscriptionTargetKey); + const voiceTranscriptionOriginTargetKeyRef = useRef(null); + voiceTranscriptionTargetKeyRef.current = voiceTranscriptionTargetKey; - const handleSend = useCallback(async () => { + const sendCurrentDraft = useCallback(async () => { const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id); if (inFlightThreadIdsRef.current.has(threadKey)) return; inFlightThreadIdsRef.current.add(threadKey); @@ -569,6 +580,68 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer props.selectedThread.id, props.selectedThread.title, ]); + const voiceTranscriptionSettings = useMobileVoiceTranscriptionSettings(); + const voiceTranscriptionConfig = activeMobileVoiceTranscriptionConfig(voiceTranscriptionSettings); + const voiceTranscriptionSendDisabled = showStopAction; + const appendVoiceTranscriptToDraft = useCallback( + (transcript: string) => { + if ( + voiceTranscriptionOriginTargetKeyRef.current === null || + voiceTranscriptionOriginTargetKeyRef.current !== voiceTranscriptionTargetKeyRef.current + ) { + return false; + } + const nextDraft = appendVoiceTranscriptText(draftMessage, transcript); + if (nextDraft === draftMessage) return false; + setComposerSelection({ start: nextDraft.length, end: nextDraft.length }); + onChangeDraftMessage(nextDraft); + voiceTranscriptionOriginTargetKeyRef.current = null; + return true; + }, + [draftMessage, onChangeDraftMessage], + ); + const voiceTranscription = useMobileVoiceTranscription({ + ...voiceTranscriptionConfig, + onTranscriptInsert: appendVoiceTranscriptToDraft, + onTranscriptSend: (transcript) => { + if (appendVoiceTranscriptToDraft(transcript) && !voiceTranscriptionSendDisabled) { + void sendCurrentDraft(); + } + }, + }); + const voiceTranscriptionReady = + voiceTranscriptionSettings.loaded && voiceTranscriptionConfig.apiKey.trim().length > 0; + const startVoiceTranscription = useCallback(async () => { + if (showStopAction) return; + voiceTranscriptionOriginTargetKeyRef.current = voiceTranscriptionTargetKeyRef.current; + await voiceTranscription.start(); + }, [showStopAction, voiceTranscription.start]); + const cancelVoiceTranscription = useCallback(async () => { + voiceTranscriptionOriginTargetKeyRef.current = null; + await voiceTranscription.cancel(); + }, [voiceTranscription.cancel]); + useEffect(() => { + const activeTargetKey = voiceTranscriptionTargetKey; + return () => { + if (voiceTranscriptionOriginTargetKeyRef.current !== activeTargetKey) return; + voiceTranscriptionOriginTargetKeyRef.current = null; + void voiceTranscription.cancel(); + }; + }, [voiceTranscription.cancel, voiceTranscriptionTargetKey]); + const handleSend = useCallback(async () => { + if (voiceTranscription.status === "recording") { + if (voiceTranscriptionSendDisabled) return; + await voiceTranscription.stop("send"); + return; + } + if (voiceTranscription.status === "transcribing") return; + await sendCurrentDraft(); + }, [ + sendCurrentDraft, + voiceTranscription.status, + voiceTranscription.stop, + voiceTranscriptionSendDisabled, + ]); const handleCommandSelect = useCallback( (item: ComposerCommandItem) => { if (!composerTrigger) return; @@ -771,137 +844,179 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer } } > - {/* Attachment strip — inside the card, above the text input */} - {isExpanded ? ( - 0 ? "pb-2.5" : undefined} - entering={FadeIn.duration(160)} - exiting={FadeOut.duration(120)} - > - - - ) : null} - - - void props.onNativePasteImages(uris)} - placeholder={props.placeholder} - onFocus={handleFocus} - onBlur={handleBlur} - onSubmit={handleSend} - scrollEnabled={isExpanded} - // Android: collapsed single line centers natively (gravity) in - // a pill-height box matching the send button; iOS keeps insets. - singleLineCentered={!isExpanded} - contentInsetVertical={isExpanded || Platform.OS === "android" ? 0 : 6} - style={ - isExpanded - ? { - minHeight: 72, - maxHeight: 160, - paddingHorizontal: 4, - paddingVertical: 4, - } - : { - height: 36, - } - } - textStyle={{ - ...bodyText, - color: foregroundColor, + {voiceTranscription.status !== "idle" ? ( + void cancelVoiceTranscription()} + onStop={() => void voiceTranscription.stop("insert")} + onSend={() => { + if (!voiceTranscriptionSendDisabled) void voiceTranscription.stop("send"); }} /> - - {!isExpanded && props.draftAttachments.length > 0 ? ( - - {props.draftAttachments.slice(0, 3).map((image) => ( - onPressImage(image.previewUri)}> - + {/* Attachment strip — inside the card, above the text input */} + {isExpanded ? ( + 0 ? "pb-2.5" : undefined} + entering={FadeIn.duration(160)} + exiting={FadeOut.duration(120)} + > + - - ))} - {props.draftAttachments.length > 3 ? ( - - - +{props.draftAttachments.length - 3} - - + ) : null} - - ) : null} - {!isExpanded ? ( - - {showStopAction ? ( - - ) : ( - - )} - - ) : null} - {isExpanded ? ( - - - void props.onPickDraftImages()} - showChevron={false} - /> - + + + void props.onNativePasteImages(uris)} + placeholder={props.placeholder} + onFocus={handleFocus} + onBlur={handleBlur} + onSubmit={handleSend} + scrollEnabled={isExpanded} + // Android: collapsed single line centers natively (gravity) in + // a pill-height box matching the send button; iOS keeps insets. + singleLineCentered={!isExpanded} + contentInsetVertical={isExpanded || Platform.OS === "android" ? 0 : 6} + style={ + isExpanded + ? { + minHeight: 72, + maxHeight: 160, + paddingHorizontal: 4, + paddingVertical: 4, + } + : { + height: 36, + } } - label={currentModelOption?.label ?? currentModelSelection.model} - maxWidth={152} - onPress={openSettings} + textStyle={{ + ...bodyText, + color: foregroundColor, + }} /> - {showStopAction ? ( + + {!isExpanded && props.draftAttachments.length > 0 ? ( + + {props.draftAttachments.slice(0, 3).map((image) => ( + onPressImage(image.previewUri)}> + + + ))} + {props.draftAttachments.length > 3 ? ( + + + +{props.draftAttachments.length - 3} + + + ) : null} + + ) : null} + {!isExpanded ? ( + + + {voiceTranscriptionReady ? ( + void startVoiceTranscription()} + /> + ) : null} + {showStopAction ? ( + + ) : ( + + )} + + + ) : null} + {isExpanded ? ( + + + void props.onPickDraftImages()} + showChevron={false} + /> + {voiceTranscriptionReady ? ( + void startVoiceTranscription()} + showChevron={false} + /> + ) : null} + + } + label={currentModelOption?.label ?? currentModelSelection.model} + maxWidth={152} + onPress={openSettings} + /> + {showStopAction ? ( + + ) : null} + - ) : null} - - - - ) : null} + + ) : null} + + )} + {voiceTranscription.status === "idle" && voiceTranscription.error ? ( + + {voiceTranscription.error} + + ) : null} + {/* Queue count */} {props.queueCount > 0 ? ( diff --git a/apps/mobile/src/features/voice-dictation/MobileVoiceTranscriptionPanel.tsx b/apps/mobile/src/features/voice-dictation/MobileVoiceTranscriptionPanel.tsx new file mode 100644 index 000000000000..c4342dd8d31d --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/MobileVoiceTranscriptionPanel.tsx @@ -0,0 +1,74 @@ +import { ActivityIndicator, View } from "react-native"; + +import { AppText as Text } from "../../components/AppText"; +import { ControlPill } from "../../components/ControlPill"; +import type { MobileVoiceTranscriptionStatus } from "./useMobileVoiceTranscription"; + +const WAVEFORM_BAR_IDS = Array.from({ length: 24 }, (_, index) => `voice-waveform-${index}`); + +function formatElapsed(elapsedMs: number): string { + const seconds = Math.floor(elapsedMs / 1_000); + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; +} + +export function MobileVoiceTranscriptionPanel(props: { + readonly status: Exclude; + readonly elapsedMs: number; + readonly levels: readonly number[]; + readonly sendDisabled: boolean; + readonly onCancel: () => void; + readonly onStop: () => void; + readonly onSend: () => void; +}) { + if (props.status === "transcribing") { + return ( + + + + Processing recording… + + ); + } + + return ( + + + + {WAVEFORM_BAR_IDS.map((barId, index) => ( + + ))} + + + {formatElapsed(props.elapsedMs)} + + + + + ); +} diff --git a/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.test.ts b/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.test.ts new file mode 100644 index 000000000000..a3c791d01727 --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from "vite-plus/test"; + +import { transcribeMobileVoiceRecording } from "./mobileVoiceTranscription"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("transcribeMobileVoiceRecording", () => { + it.each([ + { + provider: "openai" as const, + apiKey: " openai-secret ", + model: "gpt-4o-transcribe", + endpoint: "https://api.openai.com/v1/audio/transcriptions", + }, + { + provider: "groq" as const, + apiKey: " groq-secret ", + model: "whisper-large-v3-turbo", + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + }, + ])("sends an iPhone recording directly to $provider", async (config) => { + const entries: Array<[string, unknown]> = []; + vi.stubGlobal( + "FormData", + class { + append(name: string, value: unknown) { + entries.push([name, value]); + } + }, + ); + const fetchMock = vi.fn().mockResolvedValue(Response.json({ text: " hello " })); + + await expect( + transcribeMobileVoiceRecording("file:///recording.m4a", config, fetchMock), + ).resolves.toBe("hello"); + + expect(fetchMock).toHaveBeenCalledWith( + config.endpoint, + expect.objectContaining({ + method: "POST", + headers: { authorization: `Bearer ${config.apiKey.trim()}` }, + }), + ); + expect(entries).toEqual([ + ["model", config.model], + [ + "file", + { + uri: "file:///recording.m4a", + name: "recording.m4a", + type: "audio/mp4", + }, + ], + ]); + }); + + it("includes the provider error when the request fails", async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(Response.json({ error: { message: "Unknown model" } }, { status: 400 })); + + await expect( + transcribeMobileVoiceRecording( + "file:///recording.m4a", + { + provider: "groq", + apiKey: "groq-secret", + model: "future-model", + }, + fetchMock, + ), + ).rejects.toThrow("Unknown model"); + }); +}); diff --git a/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.ts b/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.ts new file mode 100644 index 000000000000..525fcece13f6 --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/mobileVoiceTranscription.ts @@ -0,0 +1,97 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; + +export interface MobileVoiceTranscriptionConfig { + readonly provider: VoiceTranscriptionProvider; + readonly apiKey: string; + readonly model: string; +} + +export interface MobileVoiceTranscriptionProviderConfig { + readonly id: VoiceTranscriptionProvider; + readonly label: string; + readonly endpoint: string; + readonly defaultModel: string; + readonly modelOptions: ReadonlyArray<{ + readonly id: string; + readonly label: string; + }>; +} + +export const MOBILE_VOICE_TRANSCRIPTION_PROVIDERS: ReadonlyArray = + [ + { + id: "openai", + label: "OpenAI", + endpoint: "https://api.openai.com/v1/audio/transcriptions", + defaultModel: "gpt-4o-transcribe", + modelOptions: [ + { id: "gpt-4o-transcribe", label: "GPT-4o Transcribe" }, + { id: "gpt-4o-mini-transcribe", label: "GPT-4o mini Transcribe" }, + { id: "whisper-1", label: "Whisper" }, + ], + }, + { + id: "groq", + label: "Groq", + endpoint: "https://api.groq.com/openai/v1/audio/transcriptions", + defaultModel: "whisper-large-v3-turbo", + modelOptions: [ + { id: "whisper-large-v3-turbo", label: "Whisper Large V3 Turbo" }, + { id: "whisper-large-v3", label: "Whisper Large V3" }, + ], + }, + ]; + +export function mobileVoiceTranscriptionProviderConfig( + provider: VoiceTranscriptionProvider, +): MobileVoiceTranscriptionProviderConfig { + return MOBILE_VOICE_TRANSCRIPTION_PROVIDERS.find((candidate) => candidate.id === provider)!; +} + +export async function transcribeMobileVoiceRecording( + uri: string, + config: MobileVoiceTranscriptionConfig, + fetchFn: typeof globalThis.fetch = globalThis.fetch, +): Promise { + const provider = mobileVoiceTranscriptionProviderConfig(config.provider); + const form = new FormData(); + form.append("model", config.model.trim()); + form.append("file", { + uri, + name: "recording.m4a", + type: "audio/mp4", + } as unknown as Blob); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 2 * 60 * 1_000); + try { + const response = await fetchFn(provider.endpoint, { + method: "POST", + headers: { authorization: `Bearer ${config.apiKey.trim()}` }, + body: form, + signal: controller.signal, + }); + const payload = (await response.json().catch(() => null)) as { + readonly text?: unknown; + readonly error?: { readonly message?: unknown }; + } | null; + if (!response.ok) { + throw new Error( + typeof payload?.error?.message === "string" + ? payload.error.message + : `${provider.label} rejected the transcription request.`, + ); + } + if (typeof payload?.text !== "string") { + throw new Error(`${provider.label} returned an invalid transcription response.`); + } + return payload.text.trim(); + } catch (cause) { + if (cause instanceof Error && cause.name === "AbortError") { + throw new Error("Voice transcription timed out.", { cause }); + } + throw cause; + } finally { + clearTimeout(timeout); + } +} diff --git a/apps/mobile/src/features/voice-dictation/useMobileVoiceTranscription.ts b/apps/mobile/src/features/voice-dictation/useMobileVoiceTranscription.ts new file mode 100644 index 000000000000..16c497eef63e --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/useMobileVoiceTranscription.ts @@ -0,0 +1,306 @@ +import { + resolveVoiceTranscriptionAction, + type VoiceTranscriptionAction, +} from "@t3tools/shared/voiceTranscription"; +import { + AudioModule, + RecordingPresets, + setAudioModeAsync, + useAudioRecorder, + useAudioRecorderState, +} from "expo-audio"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import { + type MobileVoiceTranscriptionConfig, + transcribeMobileVoiceRecording, +} from "./mobileVoiceTranscription"; + +const LEVEL_COUNT = 36; +const FLAT_LEVELS = Array(LEVEL_COUNT).fill(0); +const MIN_RECORDING_MS = 250; +const MAX_RECORDING_MS = 5 * 60 * 1_000; + +export type MobileVoiceTranscriptionStatus = "idle" | "recording" | "transcribing"; + +export function useMobileVoiceTranscription(input: { + readonly provider: MobileVoiceTranscriptionConfig["provider"]; + readonly apiKey: string; + readonly model: string; + readonly onTranscriptInsert: (text: string) => void; + readonly onTranscriptSend: (text: string) => void; +}) { + const recorder = useAudioRecorder({ + ...RecordingPresets.HIGH_QUALITY, + isMeteringEnabled: true, + numberOfChannels: 1, + }); + const recorderState = useAudioRecorderState(recorder, 50); + const [status, setStatus] = useState("idle"); + const [levels, setLevels] = useState(FLAT_LEVELS); + const [error, setError] = useState(null); + const statusRef = useRef(status); + const startingRef = useRef(false); + const cancelStartingRef = useRef(false); + const restartAfterCancellationRef = useRef(false); + const stopInFlightRef = useRef(false); + const terminalActionRef = useRef(null); + const transcriptionAttemptRef = useRef(0); + const timeoutRef = useRef | null>(null); + const mountedRef = useRef(true); + const configRef = useRef({ + provider: input.provider, + apiKey: input.apiKey, + model: input.model, + }); + const onTranscriptInsertRef = useRef(input.onTranscriptInsert); + const onTranscriptSendRef = useRef(input.onTranscriptSend); + const startRef = useRef<() => Promise>(async () => undefined); + const stopRef = useRef<(action?: "insert" | "send") => Promise>(async () => undefined); + statusRef.current = status; + configRef.current = { + provider: input.provider, + apiKey: input.apiKey, + model: input.model, + }; + onTranscriptInsertRef.current = input.onTranscriptInsert; + onTranscriptSendRef.current = input.onTranscriptSend; + + const clearRecordingTimeout = useCallback(() => { + if (timeoutRef.current !== null) clearTimeout(timeoutRef.current); + timeoutRef.current = null; + }, []); + + const resetAudioMode = useCallback(async () => { + await setAudioModeAsync({ allowsRecording: false, playsInSilentMode: true }).catch( + () => undefined, + ); + }, []); + + const stop = useCallback( + async (action: "insert" | "send" = "insert") => { + let transcriptionAttempt: number | null = null; + terminalActionRef.current = resolveVoiceTranscriptionAction( + terminalActionRef.current, + action, + ); + if (startingRef.current) { + // Permission and recorder preparation are already in flight. Keep the + // requested terminal action and apply it as soon as recording starts. + return; + } + if (statusRef.current !== "recording" || stopInFlightRef.current) return; + stopInFlightRef.current = true; + clearRecordingTimeout(); + try { + const beforeStop = await recorder.getStatus(); + await recorder.stop(); + const uri = recorder.uri; + await resetAudioMode(); + if (!mountedRef.current) return; + if ( + terminalActionRef.current === "abort" || + beforeStop.durationMillis < MIN_RECORDING_MS || + !uri + ) { + if (mountedRef.current) { + statusRef.current = "idle"; + setStatus("idle"); + setLevels(FLAT_LEVELS); + } + return; + } + + if (mountedRef.current) setStatus("transcribing"); + statusRef.current = "transcribing"; + transcriptionAttempt = ++transcriptionAttemptRef.current; + const text = await transcribeMobileVoiceRecording(uri, configRef.current); + if (!mountedRef.current || transcriptionAttempt !== transcriptionAttemptRef.current) { + return; + } + const finalAction = resolveVoiceTranscriptionAction(terminalActionRef.current, "insert"); + if (text && finalAction !== "abort") { + if (finalAction === "send") onTranscriptSendRef.current(text); + else onTranscriptInsertRef.current(text); + } + statusRef.current = "idle"; + setStatus("idle"); + setLevels(FLAT_LEVELS); + } catch (cause) { + if ( + transcriptionAttempt !== null && + transcriptionAttempt !== transcriptionAttemptRef.current + ) { + return; + } + await resetAudioMode(); + if (!mountedRef.current) return; + setError(cause instanceof Error ? cause.message : "Voice transcription failed."); + statusRef.current = "idle"; + setStatus("idle"); + setLevels(FLAT_LEVELS); + } finally { + if ( + transcriptionAttempt === null || + transcriptionAttempt === transcriptionAttemptRef.current + ) { + stopInFlightRef.current = false; + terminalActionRef.current = null; + } + } + }, + [clearRecordingTimeout, recorder, resetAudioMode], + ); + stopRef.current = stop; + + const cancel = useCallback(async () => { + terminalActionRef.current = "abort"; + if (startingRef.current) { + cancelStartingRef.current = true; + clearRecordingTimeout(); + statusRef.current = "idle"; + if (mountedRef.current) setStatus("idle"); + return; + } + if (statusRef.current === "transcribing") { + transcriptionAttemptRef.current += 1; + stopInFlightRef.current = false; + terminalActionRef.current = null; + statusRef.current = "idle"; + if (mountedRef.current) { + setStatus("idle"); + setLevels(FLAT_LEVELS); + } + return; + } + if (statusRef.current !== "recording" || stopInFlightRef.current) return; + stopInFlightRef.current = true; + clearRecordingTimeout(); + try { + await recorder.stop(); + } catch { + // Cancellation is best-effort; the audio is discarded either way. + } finally { + await resetAudioMode(); + statusRef.current = "idle"; + stopInFlightRef.current = false; + terminalActionRef.current = null; + if (mountedRef.current) { + setStatus("idle"); + setLevels(FLAT_LEVELS); + } + } + }, [clearRecordingTimeout, recorder, resetAudioMode]); + + const start = useCallback(async () => { + if (startingRef.current) { + if (cancelStartingRef.current) restartAfterCancellationRef.current = true; + return; + } + if (statusRef.current !== "idle") return; + if (!configRef.current.apiKey.trim()) { + setError("Save an API key in Settings first."); + return; + } + + startingRef.current = true; + cancelStartingRef.current = false; + restartAfterCancellationRef.current = false; + terminalActionRef.current = null; + setError(null); + setLevels(FLAT_LEVELS); + statusRef.current = "recording"; + setStatus("recording"); + try { + const permission = await AudioModule.requestRecordingPermissionsAsync(); + if (!permission.granted) { + throw new Error("Microphone permission was denied."); + } + if (cancelStartingRef.current || !mountedRef.current) { + startingRef.current = false; + cancelStartingRef.current = false; + statusRef.current = "idle"; + if (mountedRef.current) setStatus("idle"); + const shouldRestart = restartAfterCancellationRef.current; + restartAfterCancellationRef.current = false; + if (shouldRestart && mountedRef.current) { + queueMicrotask(() => void startRef.current()); + } + return; + } + await setAudioModeAsync({ allowsRecording: true, playsInSilentMode: true }); + await recorder.prepareToRecordAsync(); + if (cancelStartingRef.current || !mountedRef.current) { + startingRef.current = false; + cancelStartingRef.current = false; + await resetAudioMode(); + statusRef.current = "idle"; + if (mountedRef.current) setStatus("idle"); + const shouldRestart = restartAfterCancellationRef.current; + restartAfterCancellationRef.current = false; + if (shouldRestart && mountedRef.current) { + queueMicrotask(() => void startRef.current()); + } + return; + } + recorder.record(); + startingRef.current = false; + const pendingAction = terminalActionRef.current; + timeoutRef.current = setTimeout(() => { + void stopRef.current("insert"); + }, MAX_RECORDING_MS); + if (pendingAction === "insert" || pendingAction === "send") { + void stopRef.current(pendingAction); + } + } catch (cause) { + startingRef.current = false; + await resetAudioMode(); + if (cancelStartingRef.current) { + cancelStartingRef.current = false; + statusRef.current = "idle"; + if (mountedRef.current) setStatus("idle"); + const shouldRestart = restartAfterCancellationRef.current; + restartAfterCancellationRef.current = false; + if (shouldRestart && mountedRef.current) { + queueMicrotask(() => void startRef.current()); + } + return; + } + if (!mountedRef.current) return; + setError(cause instanceof Error ? cause.message : "Could not start the microphone."); + statusRef.current = "idle"; + setStatus("idle"); + } + }, [recorder, resetAudioMode]); + startRef.current = start; + + useEffect(() => { + if (status !== "recording") return; + const metering = recorderState.metering ?? -60; + const level = Math.min(1, Math.max(0, (metering + 60) / 60)); + setLevels((current) => [...current.slice(1), level]); + }, [recorderState.metering, status]); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + terminalActionRef.current = "abort"; + restartAfterCancellationRef.current = false; + transcriptionAttemptRef.current += 1; + clearRecordingTimeout(); + if (recorder.isRecording) void recorder.stop().catch(() => undefined); + void resetAudioMode(); + }; + }, [clearRecordingTimeout, recorder, resetAudioMode]); + + return { + status, + levels, + elapsedMs: recorderState.durationMillis, + error, + start, + stop, + cancel, + } as const; +} diff --git a/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.test.ts b/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.test.ts new file mode 100644 index 000000000000..d53c512eea9f --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.test.ts @@ -0,0 +1,103 @@ +import * as SecureStore from "expo-secure-store"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const secureStore = vi.hoisted(() => new Map()); + +vi.mock("expo-secure-store", () => ({ + getItemAsync: vi.fn((key: string) => Promise.resolve(secureStore.get(key) ?? null)), + setItemAsync: vi.fn((key: string, value: string) => { + secureStore.set(key, value); + return Promise.resolve(); + }), + deleteItemAsync: vi.fn((key: string) => { + secureStore.delete(key); + return Promise.resolve(); + }), +})); + +describe("mobile voice transcription settings", () => { + beforeEach(() => { + secureStore.clear(); + vi.clearAllMocks(); + vi.resetModules(); + }); + + it("migrates the existing OpenAI key and uses the current default model", async () => { + secureStore.set("t3code.voice-transcription.openai-api-key", " legacy-key "); + const settings = await import("./voiceTranscriptionSettings"); + + await settings.loadMobileVoiceTranscriptionSettings(); + + expect(settings.getMobileVoiceTranscriptionSettingsSnapshot()).toMatchObject({ + provider: "openai", + loaded: true, + providers: { + openai: { apiKey: "legacy-key", model: "gpt-4o-transcribe" }, + groq: { apiKey: "", model: "whisper-large-v3-turbo" }, + }, + }); + }); + + it("stores separate provider keys and a custom model", async () => { + const settings = await import("./voiceTranscriptionSettings"); + await settings.loadMobileVoiceTranscriptionSettings(); + + await settings.saveMobileVoiceTranscriptionSettings({ + provider: "groq", + providers: { + openai: { apiKey: "openai-key", model: "gpt-4o-transcribe" }, + groq: { apiKey: "groq-key", model: "future-groq-transcribe" }, + }, + }); + + expect( + settings.activeMobileVoiceTranscriptionConfig( + settings.getMobileVoiceTranscriptionSettingsSnapshot(), + ), + ).toEqual({ + provider: "groq", + apiKey: "groq-key", + model: "future-groq-transcribe", + }); + expect(SecureStore.setItemAsync).toHaveBeenCalledWith( + "t3code.voice-transcription.settings.v2", + JSON.stringify({ + provider: "groq", + providers: { + openai: { apiKey: "openai-key", model: "gpt-4o-transcribe" }, + groq: { apiKey: "groq-key", model: "future-groq-transcribe" }, + }, + }), + ); + }); + + it("keeps the setup editable when saved settings are invalid", async () => { + secureStore.set("t3code.voice-transcription.settings.v2", "not-json"); + const settings = await import("./voiceTranscriptionSettings"); + + await settings.loadMobileVoiceTranscriptionSettings(); + + expect(settings.getMobileVoiceTranscriptionSettingsSnapshot()).toMatchObject({ + provider: "openai", + loaded: true, + error: "Saved voice dictation settings were invalid. Save them again.", + }); + }); + + it("can retry after the secure store is temporarily unavailable", async () => { + vi.mocked(SecureStore.getItemAsync).mockRejectedValueOnce(new Error("temporarily unavailable")); + const settings = await import("./voiceTranscriptionSettings"); + + await settings.loadMobileVoiceTranscriptionSettings(); + expect(settings.getMobileVoiceTranscriptionSettingsSnapshot()).toMatchObject({ + loaded: false, + error: "Could not read the saved voice dictation settings.", + }); + + await settings.loadMobileVoiceTranscriptionSettings(); + expect(settings.getMobileVoiceTranscriptionSettingsSnapshot()).toMatchObject({ + loaded: true, + error: null, + }); + }); +}); diff --git a/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.ts b/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.ts new file mode 100644 index 000000000000..15f63c78c0b6 --- /dev/null +++ b/apps/mobile/src/features/voice-dictation/voiceTranscriptionSettings.ts @@ -0,0 +1,206 @@ +import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; +import * as SecureStore from "expo-secure-store"; +import { useEffect, useSyncExternalStore } from "react"; + +import { + type MobileVoiceTranscriptionConfig, + mobileVoiceTranscriptionProviderConfig, +} from "./mobileVoiceTranscription"; + +const LEGACY_OPENAI_API_KEY_STORAGE_KEY = "t3code.voice-transcription.openai-api-key"; +const SETTINGS_STORAGE_KEY = "t3code.voice-transcription.settings.v2"; + +export interface MobileVoiceTranscriptionProviderSettings { + readonly apiKey: string; + readonly model: string; +} + +export type MobileVoiceTranscriptionProviderSettingsMap = Readonly< + Record +>; + +export interface MobileVoiceTranscriptionSettingsSnapshot { + readonly provider: VoiceTranscriptionProvider; + readonly providers: MobileVoiceTranscriptionProviderSettingsMap; + readonly error: string | null; + readonly loaded: boolean; + readonly saving: boolean; +} + +function defaultProviderSettings(): MobileVoiceTranscriptionProviderSettingsMap { + return { + openai: { + apiKey: "", + model: mobileVoiceTranscriptionProviderConfig("openai").defaultModel, + }, + groq: { + apiKey: "", + model: mobileVoiceTranscriptionProviderConfig("groq").defaultModel, + }, + }; +} + +let snapshot: MobileVoiceTranscriptionSettingsSnapshot = { + provider: "openai", + providers: defaultProviderSettings(), + error: null, + loaded: false, + saving: false, +}; +let loadPromise: Promise | null = null; +let revision = 0; +const listeners = new Set<() => void>(); + +function publish(next: MobileVoiceTranscriptionSettingsSnapshot) { + snapshot = next; + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + listeners.add(listener); + return () => listeners.delete(listener); +} + +function normalizeProvider(value: unknown): VoiceTranscriptionProvider { + return value === "groq" ? "groq" : "openai"; +} + +function normalizeProviderSettings( + value: unknown, + provider: VoiceTranscriptionProvider, +): MobileVoiceTranscriptionProviderSettings { + const candidate = + typeof value === "object" && value !== null + ? (value as { readonly apiKey?: unknown; readonly model?: unknown }) + : null; + const model = typeof candidate?.model === "string" ? candidate.model.trim() : ""; + return { + apiKey: typeof candidate?.apiKey === "string" ? candidate.apiKey.trim() : "", + model: model || mobileVoiceTranscriptionProviderConfig(provider).defaultModel, + }; +} + +function parseStoredSettings( + value: string, +): Pick { + const parsed = JSON.parse(value) as { + readonly provider?: unknown; + readonly providers?: { readonly openai?: unknown; readonly groq?: unknown }; + }; + return { + provider: normalizeProvider(parsed.provider), + providers: { + openai: normalizeProviderSettings(parsed.providers?.openai, "openai"), + groq: normalizeProviderSettings(parsed.providers?.groq, "groq"), + }, + }; +} + +export function loadMobileVoiceTranscriptionSettings(): Promise { + if (snapshot.loaded) return Promise.resolve(); + if (loadPromise) return loadPromise; + const loadRevision = revision; + loadPromise = Promise.all([ + SecureStore.getItemAsync(SETTINGS_STORAGE_KEY), + SecureStore.getItemAsync(LEGACY_OPENAI_API_KEY_STORAGE_KEY), + ]) + .then(([storedSettings, legacyOpenAiApiKey]) => { + if (revision !== loadRevision) return; + if (storedSettings) { + try { + const parsed = parseStoredSettings(storedSettings); + publish({ ...parsed, error: null, loaded: true, saving: false }); + return; + } catch { + publish({ + provider: "openai", + providers: defaultProviderSettings(), + error: "Saved voice dictation settings were invalid. Save them again.", + loaded: true, + saving: false, + }); + return; + } + } + const providers = defaultProviderSettings(); + publish({ + provider: "openai", + providers: { + ...providers, + openai: { ...providers.openai, apiKey: legacyOpenAiApiKey?.trim() ?? "" }, + }, + error: null, + loaded: true, + saving: false, + }); + }) + .catch(() => { + if (revision !== loadRevision) return; + publish({ + provider: "openai", + providers: defaultProviderSettings(), + error: "Could not read the saved voice dictation settings.", + loaded: false, + saving: false, + }); + }) + .finally(() => { + loadPromise = null; + }); + return loadPromise; +} + +export async function saveMobileVoiceTranscriptionSettings(input: { + readonly provider: VoiceTranscriptionProvider; + readonly providers: MobileVoiceTranscriptionProviderSettingsMap; +}): Promise { + const providers: MobileVoiceTranscriptionProviderSettingsMap = { + openai: normalizeProviderSettings(input.providers.openai, "openai"), + groq: normalizeProviderSettings(input.providers.groq, "groq"), + }; + revision += 1; + const previous = snapshot; + publish({ ...snapshot, error: null, saving: true }); + try { + await SecureStore.setItemAsync( + SETTINGS_STORAGE_KEY, + JSON.stringify({ provider: input.provider, providers }), + ); + await SecureStore.deleteItemAsync(LEGACY_OPENAI_API_KEY_STORAGE_KEY).catch(() => undefined); + publish({ + provider: input.provider, + providers, + error: null, + loaded: true, + saving: false, + }); + } catch { + publish({ ...previous, error: "Could not save voice dictation settings.", saving: false }); + throw new Error("Could not save voice dictation settings."); + } +} + +export function activeMobileVoiceTranscriptionConfig( + settings: Pick, +): MobileVoiceTranscriptionConfig { + return { + provider: settings.provider, + ...settings.providers[settings.provider], + }; +} + +export function getMobileVoiceTranscriptionSettingsSnapshot(): MobileVoiceTranscriptionSettingsSnapshot { + return snapshot; +} + +export function useMobileVoiceTranscriptionSettings() { + const current = useSyncExternalStore( + subscribe, + () => snapshot, + () => snapshot, + ); + useEffect(() => { + void loadMobileVoiceTranscriptionSettings(); + }, []); + return current; +} diff --git a/apps/mobile/src/state/computers.ts b/apps/mobile/src/state/computers.ts new file mode 100644 index 000000000000..07c5f9c6f099 --- /dev/null +++ b/apps/mobile/src/state/computers.ts @@ -0,0 +1,5 @@ +import { createComputerEnvironmentAtoms } from "@t3tools/client-runtime/state/computers"; + +import { connectionAtomRuntime } from "../connection/runtime"; + +export const computerEnvironment = createComputerEnvironmentAtoms(connectionAtomRuntime); diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index a031f0c81cb5..84fa64b8e5b7 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -128,6 +128,9 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.previewAutomationConnect]: AuthOrchestrationOperateScope, [WS_METHODS.previewAutomationRespond]: AuthOrchestrationOperateScope, [WS_METHODS.previewAutomationFocusHost]: AuthOrchestrationOperateScope, + [WS_METHODS.computersConnect]: AuthOrchestrationOperateScope, + [WS_METHODS.computersSync]: AuthOrchestrationOperateScope, + [WS_METHODS.computersRespond]: AuthOrchestrationOperateScope, [WS_METHODS.subscribePreviewEvents]: AuthOrchestrationReadScope, [WS_METHODS.subscribeDiscoveredLocalServers]: AuthOrchestrationReadScope, [WS_METHODS.subscribeServerConfig]: AuthOrchestrationReadScope, diff --git a/apps/server/src/mcp/ComputerTaskBroker.ts b/apps/server/src/mcp/ComputerTaskBroker.ts new file mode 100644 index 000000000000..200a5037e1b8 --- /dev/null +++ b/apps/server/src/mcp/ComputerTaskBroker.ts @@ -0,0 +1,338 @@ +import { + COMPUTER_SEND_TIMEOUT_MS, + ComputerTaskError, + type ComputerListEntry, + type ComputerPeer, + type ComputerTaskHost, + type ComputerTaskResponse, + type ComputerTaskSendRequest, + type ComputerTaskSendResult, + type ComputerTaskStreamEvent, + type ExecutionEnvironmentDescriptor, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as SynchronizedRef from "effect/SynchronizedRef"; + +export class ComputerTaskBroker extends Context.Service< + ComputerTaskBroker, + { + readonly connect: ( + host: ComputerTaskHost, + ) => Effect.Effect>; + readonly sync: (host: ComputerTaskHost) => Effect.Effect; + readonly respond: (response: ComputerTaskResponse) => Effect.Effect; + readonly list: ( + descriptor: ExecutionEnvironmentDescriptor, + ) => Effect.Effect>; + readonly send: ( + request: Omit, + ) => Effect.Effect; + } +>()("t3/mcp/ComputerTaskBroker") {} + +interface ClientConnection { + readonly clientId: string; + readonly connectionId: string; + readonly computers: ReadonlyArray; + readonly queue: Queue.Queue; +} + +interface PendingRequest { + readonly deferred: Deferred.Deferred; + readonly clientId: string; + readonly connectionId: string; +} + +interface BrokerState { + readonly clients: ReadonlyMap; + readonly pending: ReadonlyMap; +} + +const removeConnection = ( + current: BrokerState, + clientId: string, + queue: ClientConnection["queue"], +): { readonly state: BrokerState; readonly disconnected: ReadonlyArray } => { + const existing = current.clients.get(clientId); + if (!existing || existing.queue !== queue) { + return { state: current, disconnected: [] }; + } + const clients = new Map(current.clients); + clients.delete(clientId); + const disconnected: Array = []; + const pending = new Map(current.pending); + for (const [requestId, entry] of current.pending) { + if (entry.clientId === clientId && entry.connectionId === existing.connectionId) { + pending.delete(requestId); + disconnected.push(entry); + } + } + return { state: { clients, pending }, disconnected }; +}; + +const toListEntry = (peer: ComputerPeer, thisEnvironmentId: string): ComputerListEntry => ({ + ...peer, + thisMachine: peer.environmentId === thisEnvironmentId, +}); + +export const mergeComputerCatalog = ( + descriptor: ExecutionEnvironmentDescriptor, + catalogs: ReadonlyArray>, +): ReadonlyArray => { + const byId = new Map(); + byId.set(descriptor.environmentId, { + environmentId: descriptor.environmentId, + label: descriptor.label, + kind: "local", + os: descriptor.platform.os, + connected: true, + }); + for (const catalog of catalogs) { + for (const peer of catalog) { + const existing = byId.get(peer.environmentId); + if ( + !existing || + (peer.connected && !existing.connected) || + (peer.connected === existing.connected && + peer.kind !== "local" && + existing.kind === "local") + ) { + byId.set(peer.environmentId, peer); + } + } + } + return [...byId.values()] + .map((peer) => toListEntry(peer, descriptor.environmentId)) + .toSorted((left, right) => { + if (left.thisMachine !== right.thisMachine) return left.thisMachine ? -1 : 1; + if (left.connected !== right.connected) return left.connected ? -1 : 1; + return left.label.localeCompare(right.label); + }); +}; + +const pickClientForComputer = ( + clients: ReadonlyMap, + environmentId: string, +): ClientConnection | undefined => { + const advertised = [...clients.values()].filter((client) => + client.computers.some((computer) => computer.environmentId === environmentId), + ); + return ( + advertised.find((client) => + client.computers.some( + (computer) => computer.environmentId === environmentId && computer.connected, + ), + ) ?? + advertised[0] ?? + [...clients.values()][0] + ); +}; + +export const make = Effect.gen(function* ComputerTaskBrokerMake() { + const crypto = yield* Crypto.Crypto; + const state = yield* SynchronizedRef.make({ + clients: new Map(), + pending: new Map(), + }); + + const closeConnection = Effect.fn("ComputerTaskBroker.closeConnection")(function* ( + queue: ClientConnection["queue"], + disconnected: ReadonlyArray, + ) { + yield* Effect.forEach( + disconnected, + ({ deferred }) => + Deferred.fail( + deferred, + new ComputerTaskError({ + code: "no_client", + detail: "The T3 Code client that could reach that computer disconnected.", + }), + ), + { discard: true }, + ); + yield* Queue.shutdown(queue); + }); + + const disconnect = Effect.fn("ComputerTaskBroker.disconnect")(function* ( + clientId: string, + queue: ClientConnection["queue"], + ) { + const disconnected = yield* SynchronizedRef.modify(state, (current) => { + const removed = removeConnection(current, clientId, queue); + return [removed.disconnected, removed.state] as const; + }); + yield* closeConnection(queue, disconnected); + }); + + const acquireConnection = Effect.fn("ComputerTaskBroker.acquireConnection")(function* ( + host: ComputerTaskHost, + ) { + const queue = yield* Queue.unbounded(); + const connectionId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + yield* Queue.offer(queue, { type: "connected", connectionId }); + const connection: ClientConnection = { + clientId: host.clientId, + connectionId, + computers: host.computers, + queue, + }; + const registration = yield* SynchronizedRef.modify(state, (current) => { + const previous = current.clients.get(host.clientId); + const removed = previous + ? removeConnection(current, host.clientId, previous.queue) + : { state: current, disconnected: [] }; + const clients = new Map(removed.state.clients); + clients.set(host.clientId, connection); + return [ + { previous, disconnected: removed.disconnected }, + { ...removed.state, clients }, + ] as const; + }); + if (registration.previous) { + yield* closeConnection(registration.previous.queue, registration.disconnected); + } + return connection; + }); + + const connect: ComputerTaskBroker["Service"]["connect"] = Effect.fn("ComputerTaskBroker.connect")( + (host) => + Effect.succeed( + Stream.unwrap( + Effect.acquireRelease(acquireConnection(host), (connection) => + disconnect(connection.clientId, connection.queue), + ).pipe(Effect.map((connection) => Stream.fromQueue(connection.queue))), + ), + ), + ); + + const sync: ComputerTaskBroker["Service"]["sync"] = Effect.fn("ComputerTaskBroker.sync")( + function* (host) { + const updated = yield* SynchronizedRef.modify(state, (current) => { + const existing = current.clients.get(host.clientId); + if (!existing) return [false, current] as const; + const clients = new Map(current.clients); + clients.set(host.clientId, { ...existing, computers: host.computers }); + return [true, { ...current, clients }] as const; + }); + if (!updated) { + return yield* new ComputerTaskError({ + code: "no_client", + detail: "No computer-task client is connected with that id.", + }); + } + }, + ); + + const respond: ComputerTaskBroker["Service"]["respond"] = Effect.fn("ComputerTaskBroker.respond")( + function* (response) { + const pending = yield* SynchronizedRef.modify(state, (current) => { + const entry = current.pending.get(response.requestId); + if ( + !entry || + entry.clientId !== response.clientId || + entry.connectionId !== response.connectionId + ) { + return [undefined, current] as const; + } + const pending = new Map(current.pending); + pending.delete(response.requestId); + return [entry, { ...current, pending }] as const; + }); + if (!pending) return; + if (response.ok && response.result) { + yield* Deferred.succeed(pending.deferred, response.result); + return; + } + yield* Deferred.fail( + pending.deferred, + new ComputerTaskError({ + code: response.error?.code ?? "dispatch_failed", + detail: + response.error?.detail ?? + "The T3 Code client could not start the task on that computer.", + }), + ); + }, + ); + + const list: ComputerTaskBroker["Service"]["list"] = Effect.fn("ComputerTaskBroker.list")( + function* (descriptor) { + const current = yield* SynchronizedRef.get(state); + return mergeComputerCatalog( + descriptor, + [...current.clients.values()].map((client) => client.computers), + ); + }, + ); + + const send: ComputerTaskBroker["Service"]["send"] = Effect.fn("ComputerTaskBroker.send")( + function* (request) { + const requestId = yield* crypto.randomUUIDv4.pipe(Effect.orDie); + const deferred = yield* Deferred.make(); + const route = yield* SynchronizedRef.modify(state, (current) => { + const client = pickClientForComputer(current.clients, request.computer.environmentId); + if (!client) return [undefined, current] as const; + const pending = new Map(current.pending); + pending.set(requestId, { + deferred, + clientId: client.clientId, + connectionId: client.connectionId, + }); + return [ + { client, requestId }, + { ...current, pending }, + ] as const; + }); + if (!route) { + return yield* new ComputerTaskError({ + code: "no_client", + detail: + "No T3 Code client is connected that can reach another computer. Keep the desktop or web app open.", + }); + } + const offered = yield* Queue.offer(route.client.queue, { + type: "request", + connectionId: route.client.connectionId, + request: { ...request, requestId: route.requestId }, + }); + const removePending = SynchronizedRef.update(state, (current) => { + const pending = new Map(current.pending); + pending.delete(route.requestId); + return { ...current, pending }; + }); + if (!offered) { + yield* removePending; + return yield* new ComputerTaskError({ + code: "no_client", + detail: "The T3 Code client disconnected before the task could be sent.", + }); + } + const result = yield* Deferred.await(deferred).pipe( + Effect.timeoutOption(COMPUTER_SEND_TIMEOUT_MS), + Effect.ensuring(removePending), + ); + return yield* Option.match(result, { + onNone: () => + Effect.fail( + new ComputerTaskError({ + code: "dispatch_failed", + detail: "Timed out waiting for the other computer to accept the task.", + }), + ), + onSome: (value) => Effect.succeed(value), + }); + }, + ); + + return ComputerTaskBroker.of({ connect, sync, respond, list, send }); +}).pipe(Effect.withSpan("ComputerTaskBroker.make")); + +export const layer = Layer.effect(ComputerTaskBroker, make); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index bccca00d4d45..26709e54e9b4 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -26,6 +26,8 @@ import { ThreadReferenceToolkitHandlersLive } from "./toolkits/threadReference/h import { ThreadReferenceToolkit } from "./toolkits/threadReference/tools.ts"; import { ThreadRelayToolkitHandlersLive } from "./toolkits/threads/handlers.ts"; import { ThreadRelayToolkit } from "./toolkits/threads/tools.ts"; +import { ComputerToolkitHandlersLive } from "./toolkits/computers/handlers.ts"; +import { ComputerToolkit } from "./toolkits/computers/tools.ts"; const unauthorized = HttpServerResponse.jsonUnsafe( { @@ -228,6 +230,10 @@ export const ThreadRelayToolkitRegistrationLive = McpServer.toolkit(ThreadRelayT Layer.provide(ThreadRelayToolkitHandlersLive), ); +export const ComputerToolkitRegistrationLive = McpServer.toolkit(ComputerToolkit).pipe( + Layer.provide(ComputerToolkitHandlersLive), +); + const McpTransportLive = McpServer.layerHttp({ name: "T3 Code", version: packageJson.version, @@ -239,4 +245,5 @@ export const layer = Layer.mergeAll( PreviewToolkitRegistrationLive, ThreadReferenceToolkitRegistrationLive, ThreadRelayToolkitRegistrationLive, + ComputerToolkitRegistrationLive, ).pipe(Layer.provideMerge(McpTransportLive)); diff --git a/apps/server/src/mcp/toolkits/computers/handlers.test.ts b/apps/server/src/mcp/toolkits/computers/handlers.test.ts new file mode 100644 index 000000000000..37a97725bb51 --- /dev/null +++ b/apps/server/src/mcp/toolkits/computers/handlers.test.ts @@ -0,0 +1,203 @@ +import { expect, it } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + EnvironmentId, + type OrchestrationCommand, + type OrchestrationShellSnapshot, + type OrchestrationThreadShell, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; +import { McpSchema, McpServer } from "effect/unstable/ai"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as ComputerTaskBroker from "../../ComputerTaskBroker.ts"; +import * as ServerEnvironment from "../../../environment/ServerEnvironment.ts"; +import { OrchestrationEngineService } from "../../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ComputerToolkitHandlersLive } from "./handlers.ts"; +import { ComputerToolkit } from "./tools.ts"; + +const now = "2026-01-01T00:00:00.000Z"; +const environmentId = EnvironmentId.make("environment-mac"); +const projectId = ProjectId.make("project-t3"); +const sourceThreadId = ThreadId.make("thread-source"); + +function makeThread( + id: ThreadId, + input: Partial = {}, +): OrchestrationThreadShell { + return { + id, + projectId, + title: "Source Agent", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: null, + latestTurn: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + session: null, + latestUserMessageAt: null, + hasPendingApprovals: false, + hasPendingUserInput: false, + hasActionableProposedPlan: false, + ...input, + }; +} + +const source = makeThread(sourceThreadId); +const snapshot = { + snapshotSequence: 10, + projects: [ + { + id: projectId, + title: "t3code", + workspaceRoot: "/Users/me/dev/t3code", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + ], + threads: [source], + updatedAt: now, +} satisfies OrchestrationShellSnapshot; + +const invocation = { + environmentId, + threadId: sourceThreadId, + providerSessionId: "provider-session-computers", + providerInstanceId: ProviderInstanceId.make("codex"), + capabilities: new Set(), + issuedAt: 1, +}; + +const client = McpSchema.McpServerClient.of({ + clientId: 1, + protocolVersion: "2025-06-18", + initializePayload: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "computer-task-test", version: "1.0.0" }, + }, + getClient: Effect.die("unused"), +}); + +const descriptor = { + environmentId, + label: "Sheehan's Mac", + platform: { os: "darwin" as const, arch: "arm64" as const }, + serverVersion: "0.0.1", + capabilities: { repositoryIdentity: false }, +}; + +function makeTestLayer(dispatched: Array) { + const query = { + getShellSnapshot: () => Effect.succeed(snapshot), + getThreadShellById: (threadId: ThreadId) => { + const thread = snapshot.threads.find(({ id }) => id === threadId); + return Effect.succeed(thread === undefined ? Option.none() : Option.some(thread)); + }, + } as unknown as ProjectionSnapshotQuery["Service"]; + + const engine = { + readEvents: () => Stream.empty, + dispatch: (command) => { + dispatched.push(command); + return Effect.succeed({ sequence: 11 }); + }, + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(10), + } satisfies OrchestrationEngineService["Service"]; + + const environment = { + getEnvironmentId: Effect.succeed(environmentId), + getDescriptor: Effect.succeed(descriptor), + } satisfies ServerEnvironment.ServerEnvironment["Service"]; + + return McpServer.toolkit(ComputerToolkit).pipe( + Layer.provide(ComputerToolkitHandlersLive), + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provideMerge(ComputerTaskBroker.layer), + Layer.provideMerge(Layer.succeed(ProjectionSnapshotQuery, query)), + Layer.provideMerge(Layer.succeed(OrchestrationEngineService, engine)), + Layer.provideMerge(Layer.succeed(ServerEnvironment.ServerEnvironment, environment)), + Layer.provideMerge(NodeServices.layer), + ); +} + +it.effect("lists this machine and starts a local thread for computer_send this", () => { + const dispatched: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + expect(server.tools.some(({ tool }) => tool.name === "computer_list")).toBe(true); + expect(server.tools.some(({ tool }) => tool.name === "computer_send")).toBe(true); + + const listed = yield* server + .callTool({ name: "computer_list", arguments: {} }) + .pipe( + Effect.provideService(McpSchema.McpServerClient, client), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + ); + expect(listed.isError).toBe(false); + expect(listed.structuredContent).toMatchObject({ + thisEnvironmentId: environmentId, + computers: [{ label: "Sheehan's Mac", thisMachine: true }], + }); + + const sent = yield* server + .callTool({ + name: "computer_send", + arguments: { computer: "this", message: "Build the Windows installer." }, + }) + .pipe( + Effect.provideService(McpSchema.McpServerClient, client), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + ); + expect(sent.isError).toBe(false); + expect(dispatched[0]?.type).toBe("thread.turn.start"); + if (dispatched[0]?.type === "thread.turn.start") { + expect(dispatched[0].message.text).toContain("Build the Windows installer."); + expect(dispatched[0].bootstrap?.createThread?.projectId).toBe(projectId); + } + }), + ).pipe(Effect.provide(makeTestLayer(dispatched))); +}); + +it.effect("refuses an unknown computer", () => { + const dispatched: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const sent = yield* server + .callTool({ + name: "computer_send", + arguments: { computer: "toaster", message: "hello" }, + }) + .pipe( + Effect.provideService(McpSchema.McpServerClient, client), + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + ); + expect(sent.isError).toBe(true); + const text = sent.content.find((part) => part.type === "text"); + expect(text?.type === "text" && text.text.includes("No computer matches 'toaster'")).toBe( + true, + ); + }), + ).pipe(Effect.provide(makeTestLayer(dispatched))); +}); diff --git a/apps/server/src/mcp/toolkits/computers/handlers.ts b/apps/server/src/mcp/toolkits/computers/handlers.ts new file mode 100644 index 000000000000..13bdf1a8022b --- /dev/null +++ b/apps/server/src/mcp/toolkits/computers/handlers.ts @@ -0,0 +1,220 @@ +import { + CommandId, + ComputerTaskError, + MessageId, + ThreadId, + type OrchestrationThreadShell, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as ComputerTaskBroker from "../../ComputerTaskBroker.ts"; +import * as ServerEnvironment from "../../../environment/ServerEnvironment.ts"; +import { OrchestrationEngineService } from "../../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { resolveComputer } from "./resolve.ts"; +import { ComputerToolkit } from "./tools.ts"; + +const isComputerTaskError = Schema.is(ComputerTaskError); +const TITLE_MAX = 80; + +function threadTitle(preferred: string | undefined, message: string, sourceLabel: string): string { + const fromPreferred = preferred?.trim(); + if (fromPreferred && fromPreferred.length > 0) return fromPreferred.slice(0, TITLE_MAX); + const firstLine = message.split(/\r?\n/, 1)[0]?.trim() ?? ""; + if (firstLine.length > 0) return firstLine.slice(0, TITLE_MAX); + return `Task from ${sourceLabel}`.slice(0, TITLE_MAX); +} + +export function formatComputerTaskMessage(input: { + readonly sourceLabel: string; + readonly sourceEnvironmentId: string; + readonly sourceThreadId: string; + readonly message: string; +}): string { + return [ + "[T3 computer task — server-authored]", + `From: ${input.sourceLabel} (${input.sourceEnvironmentId})`, + `Source thread: ${input.sourceThreadId}`, + `Reply with computer_send to ${input.sourceLabel} only if useful.`, + "", + input.message, + ].join("\n"); +} + +const readActiveThread = Effect.fn("ComputerTask.readActiveThread")(function* ( + query: ProjectionSnapshotQuery["Service"], + threadId: OrchestrationThreadShell["id"], +) { + const thread = yield* query.getThreadShellById(threadId).pipe( + Effect.mapError( + (cause) => + new ComputerTaskError({ + code: "query_failed", + detail: "T3 could not read the current thread.", + cause, + }), + ), + ); + return Option.filter(thread, ({ archivedAt }) => archivedAt === null); +}); + +const handlers = { + computer_list: Effect.fn("ComputerTask.computerList")(function* () { + const broker = yield* ComputerTaskBroker.ComputerTaskBroker; + const environment = yield* ServerEnvironment.ServerEnvironment; + const descriptor = yield* environment.getDescriptor; + const computers = yield* broker.list(descriptor); + return { + thisEnvironmentId: descriptor.environmentId, + computers, + }; + }), + computer_send: Effect.fn("ComputerTask.computerSend")(function* (input) { + const invocation = yield* McpInvocationContext.McpInvocationContext; + const query = yield* ProjectionSnapshotQuery; + const source = yield* readActiveThread(query, invocation.threadId); + if (Option.isNone(source)) { + return yield* new ComputerTaskError({ + code: "source_unavailable", + detail: "The invoking T3 thread is no longer active.", + }); + } + + const snapshot = yield* query.getShellSnapshot().pipe( + Effect.mapError( + (cause) => + new ComputerTaskError({ + code: "query_failed", + detail: "T3 could not read the current project catalog.", + cause, + }), + ), + ); + const sourceProject = snapshot.projects.find( + (project) => project.id === source.value.projectId, + ); + if (!sourceProject) { + return yield* new ComputerTaskError({ + code: "source_unavailable", + detail: "The invoking T3 thread's project is no longer available.", + }); + } + + const broker = yield* ComputerTaskBroker.ComputerTaskBroker; + const environment = yield* ServerEnvironment.ServerEnvironment; + const descriptor = yield* environment.getDescriptor; + const computers = yield* broker.list(descriptor); + const resolved = resolveComputer(input.computer, computers); + if (isComputerTaskError(resolved)) return yield* resolved; + + const title = threadTitle(input.title, input.message, descriptor.label); + const message = formatComputerTaskMessage({ + sourceLabel: descriptor.label, + sourceEnvironmentId: descriptor.environmentId, + sourceThreadId: source.value.id, + message: input.message, + }); + + if (resolved.thisMachine) { + const crypto = yield* Crypto.Crypto; + const engine = yield* OrchestrationEngineService; + const [commandUuid, threadUuid, messageUuid, createdAt] = yield* Effect.all([ + crypto.randomUUIDv4, + crypto.randomUUIDv4, + crypto.randomUUIDv4, + Effect.map(DateTime.now, DateTime.formatIso), + ]).pipe( + Effect.mapError( + (cause) => + new ComputerTaskError({ + code: "dispatch_failed", + detail: "T3 could not start a thread on this computer.", + cause, + }), + ), + ); + const threadId = ThreadId.make(threadUuid); + const accepted = yield* engine + .dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`mcp:computer-send:${commandUuid}`), + threadId, + message: { + messageId: MessageId.make(messageUuid), + role: "user", + text: message, + attachments: [], + }, + modelSelection: source.value.modelSelection, + titleSeed: title, + runtimeMode: source.value.runtimeMode, + interactionMode: source.value.interactionMode, + bootstrap: { + createThread: { + projectId: source.value.projectId, + title, + modelSelection: source.value.modelSelection, + runtimeMode: source.value.runtimeMode, + interactionMode: source.value.interactionMode, + branch: null, + worktreePath: null, + createdAt, + }, + }, + createdAt, + }) + .pipe( + Effect.mapError( + (cause) => + new ComputerTaskError({ + code: "dispatch_failed", + detail: "T3 could not start a thread on this computer.", + cause, + }), + ), + ); + void accepted; + return { + environmentId: descriptor.environmentId, + threadId, + projectId: source.value.projectId, + }; + } + + return yield* broker.send({ + computer: { + environmentId: resolved.environmentId, + label: resolved.label, + kind: resolved.kind, + os: resolved.os, + connected: resolved.connected, + ...(resolved.sshTarget === undefined ? {} : { sshTarget: resolved.sshTarget }), + }, + message, + title, + source: { + environmentId: descriptor.environmentId, + threadId: source.value.id, + label: descriptor.label, + projectTitle: sourceProject.title, + projectWorkspaceRoot: sourceProject.workspaceRoot, + }, + projectHint: input.project ?? null, + modelSelection: source.value.modelSelection, + runtimeMode: source.value.runtimeMode, + interactionMode: source.value.interactionMode, + }); + }), +} satisfies Parameters[0]; + +export const ComputerToolkitHandlersLive = ComputerToolkit.toLayer(handlers); + +export const __testing = { + formatComputerTaskMessage, + threadTitle, +}; diff --git a/apps/server/src/mcp/toolkits/computers/resolve.test.ts b/apps/server/src/mcp/toolkits/computers/resolve.test.ts new file mode 100644 index 000000000000..4b0f70b11ee1 --- /dev/null +++ b/apps/server/src/mcp/toolkits/computers/resolve.test.ts @@ -0,0 +1,89 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; + +import { mergeComputerCatalog } from "../../ComputerTaskBroker.ts"; +import { resolveComputer } from "./resolve.ts"; + +const mac = EnvironmentId.make("env-mac"); +const blade = EnvironmentId.make("env-blade"); + +const computers = [ + { + environmentId: mac, + label: "Sheehan's Mac", + kind: "local" as const, + os: "darwin" as const, + connected: true, + thisMachine: true, + }, + { + environmentId: blade, + label: "Blade", + kind: "ssh" as const, + os: "windows" as const, + connected: true, + thisMachine: false, + sshTarget: "muhha@192.168.50.64", + }, +]; + +describe("resolveComputer", () => { + it("resolves this/here to the local machine", () => { + const resolved = resolveComputer("this", computers); + expect("thisMachine" in resolved && resolved.thisMachine).toBe(true); + }); + + it("resolves a label or SSH host", () => { + expect(resolveComputer("Blade", computers)).toMatchObject({ environmentId: blade }); + expect(resolveComputer("muhha@192.168.50.64", computers)).toMatchObject({ + environmentId: blade, + }); + }); + + it("rejects unknown names", () => { + const resolved = resolveComputer("toaster", computers); + expect(resolved).toMatchObject({ _tag: "ComputerTaskError", code: "computer_not_found" }); + }); +}); + +describe("mergeComputerCatalog", () => { + it("always includes this machine and prefers connected peer rows", () => { + const merged = mergeComputerCatalog( + { + environmentId: mac, + label: "Mac", + platform: { os: "darwin", arch: "arm64" }, + serverVersion: "0.0.1", + capabilities: { repositoryIdentity: false }, + }, + [ + [ + { + environmentId: blade, + label: "Blade", + kind: "ssh", + os: "windows", + connected: false, + }, + ], + [ + { + environmentId: blade, + label: "Blade", + kind: "ssh", + os: "windows", + connected: true, + sshTarget: "muhha@blade", + }, + ], + ], + ); + expect(merged[0]).toMatchObject({ environmentId: mac, thisMachine: true, connected: true }); + expect(merged[1]).toMatchObject({ + environmentId: blade, + connected: true, + sshTarget: "muhha@blade", + thisMachine: false, + }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/computers/resolve.ts b/apps/server/src/mcp/toolkits/computers/resolve.ts new file mode 100644 index 000000000000..8e3f994f1741 --- /dev/null +++ b/apps/server/src/mcp/toolkits/computers/resolve.ts @@ -0,0 +1,60 @@ +import type { ComputerListEntry } from "@t3tools/contracts"; +import { ComputerTaskError } from "@t3tools/contracts"; + +const THIS_ALIASES = new Set(["this", "here", "this machine", "this computer", "local"]); + +function normalize(value: string): string { + return value.trim().toLowerCase(); +} + +export function resolveComputer( + query: string, + computers: ReadonlyArray, +): ComputerListEntry | ComputerTaskError { + const needle = normalize(query); + if (needle.length === 0) { + return new ComputerTaskError({ + code: "computer_not_found", + detail: "Pass a computer id, label, SSH host, or 'this'.", + }); + } + + const thisMachine = computers.find((computer) => computer.thisMachine); + if (THIS_ALIASES.has(needle) && thisMachine) return thisMachine; + + const exactId = computers.find((computer) => computer.environmentId === query.trim()); + if (exactId) return exactId; + + const exactLabel = computers.filter((computer) => normalize(computer.label) === needle); + if (exactLabel.length === 1) return exactLabel[0]!; + if (exactLabel.length > 1) { + return new ComputerTaskError({ + code: "computer_not_found", + detail: `Multiple computers are named '${query}'. Use an environment id from computer_list.`, + }); + } + + const sshMatches = computers.filter((computer) => { + const ssh = computer.sshTarget; + return ssh !== undefined && (normalize(ssh) === needle || ssh.toLowerCase().includes(needle)); + }); + if (sshMatches.length === 1) return sshMatches[0]!; + + const prefix = computers.filter( + (computer) => + normalize(computer.label).startsWith(needle) || + computer.environmentId.toLowerCase().startsWith(needle), + ); + if (prefix.length === 1) return prefix[0]!; + + const available = computers + .map( + (computer) => + `${computer.label} (${computer.kind}${computer.connected ? "" : ", offline"}${computer.thisMachine ? ", this machine" : ""})`, + ) + .join("; "); + return new ComputerTaskError({ + code: "computer_not_found", + detail: `No computer matches '${query}'. Available: ${available || "only this machine"}.`, + }); +} diff --git a/apps/server/src/mcp/toolkits/computers/tools.ts b/apps/server/src/mcp/toolkits/computers/tools.ts new file mode 100644 index 000000000000..5846ab5d245a --- /dev/null +++ b/apps/server/src/mcp/toolkits/computers/tools.ts @@ -0,0 +1,77 @@ +import { + COMPUTER_SEND_MESSAGE_MAX_CHARS, + ComputerListResult, + ComputerTaskError, + ComputerTaskSendResult, + TrimmedNonEmptyString, +} from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as Schema from "effect/Schema"; +import { Tool, Toolkit } from "effect/unstable/ai"; + +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as ComputerTaskBroker from "../../ComputerTaskBroker.ts"; +import * as ServerEnvironment from "../../../environment/ServerEnvironment.ts"; +import { OrchestrationEngineService } from "../../../orchestration/Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../../../orchestration/Services/ProjectionSnapshotQuery.ts"; + +const dependencies = [ + McpInvocationContext.McpInvocationContext, + ComputerTaskBroker.ComputerTaskBroker, + ServerEnvironment.ServerEnvironment, + ProjectionSnapshotQuery, + OrchestrationEngineService, + Crypto.Crypto, +]; + +const ComputerSendInput = Schema.Struct({ + computer: TrimmedNonEmptyString.annotate({ + description: + "Target computer: environment id, label, SSH host (user@host), or 'this' for the machine this chat is already on.", + }), + message: TrimmedNonEmptyString.check( + Schema.isMaxLength(COMPUTER_SEND_MESSAGE_MAX_CHARS), + ).annotate({ + description: + "Full task for the agent on the target computer. Include all context it needs; this chat's transcript is not shared.", + }), + project: Schema.optional( + TrimmedNonEmptyString.annotate({ + description: + "Optional project title or folder name on the target computer. Omit to match this thread's project name.", + }), + ), + title: Schema.optional( + TrimmedNonEmptyString.annotate({ + description: "Optional title for the new thread on the target computer.", + }), + ), +}); + +export const ComputerListTool = Tool.make("computer_list", { + description: + "List computers this T3 Code client can run work on: this machine, SSH hosts, T3 Connect machines, and other paired environments. Call this before computer_send when the task belongs on another OS, desktop, GPU, or filesystem. Returns ids, labels, OS, connection kind, and whether each computer is reachable.", + parameters: Schema.Struct({}), + success: ComputerListResult, + failure: ComputerTaskError, + dependencies, +}) + .annotate(Tool.Title, "List computers") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true); + +export const ComputerSendTool = Tool.make("computer_send", { + description: + "Start a new T3 thread on another connected computer (or this one) with the given task. The receiving agent runs on that machine, so it has that computer's files, terminal, and Computer Use desktop. Use computer_list to discover ids and labels. This does not move this chat or share its transcript; include everything the recipient needs.", + parameters: ComputerSendInput, + success: ComputerTaskSendResult, + failure: ComputerTaskError, + dependencies, +}) + .annotate(Tool.Title, "Send task to computer") + .annotate(Tool.Readonly, false) + .annotate(Tool.Destructive, true) + .annotate(Tool.Idempotent, false); + +export const ComputerToolkit = Toolkit.make(ComputerListTool, ComputerSendTool); diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index 20a1f617e5f2..ae62ccfdcc3f 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -28,6 +28,15 @@ const T3_CODE_THREAD_REFERENCE_INSTRUCTIONS = ` The user may reference another chat with a \`t3-thread\` link. Do not treat the link as a web URL and do not assume its contents. Use the read-only \`thread_read\` tool from the \`t3-code\` MCP server. For \`t3-thread:///ENVIRONMENT_ID/THREAD_ID\`, pass the final \`THREAD_ID\` path segment as \`threadId\`; the tool also accepts the full link. Follow \`nextCursor\` until you have read as much of the transcript as the task requires. `; +const T3_CODE_COMPUTER_TOOL_INSTRUCTIONS = ` + +## T3 Code computers + +You are running on one T3 Code environment (one computer). Other computers the user has connected — this machine, SSH hosts, T3 Connect, or paired LAN environments — are available through the \`t3-code\` MCP tools \`computer_list\` and \`computer_send\`. + +If the task needs another computer's desktop, OS, files, GPU, or a machine that is already linked in T3 Code, call \`computer_list\` and then \`computer_send\` instead of asking the user to switch the Run on picker. Include the full task in \`message\`; the other computer does not receive this transcript. Use \`computer\` set to an id, label, SSH host, or \`this\`. +`; + export const codexPlanModeDeveloperInstructions = ( browserToolsAvailable: boolean, ): string => `# Plan Mode (Conversational) @@ -160,6 +169,7 @@ Only produce at most one \`\` block per turn, and only when you a If the user stays in Plan mode and asks for revisions after a prior \`\`, any new \`\` must be a complete replacement. If the user indicates that the prior plan is not acceptable but does not provide enough information to produce a complete replacement, address the concern and continue planning without producing a \`\` block. If the follow-up neither requires changes nor calls the plan into question (e.g. clarifying question), answer it before the block, then reproduce the prior \`\` unchanged. ${browserToolInstructions(browserToolsAvailable)} ${T3_CODE_THREAD_REFERENCE_INSTRUCTIONS} +${T3_CODE_COMPUTER_TOOL_INSTRUCTIONS} `; export const codexDefaultModeDeveloperInstructions = ( @@ -177,6 +187,7 @@ Use the \`request_user_input\` tool only when it is listed in the available tool In Default mode, strongly prefer making reasonable assumptions and executing the user's request rather than stopping to ask questions. If you absolutely must ask a question because the answer cannot be discovered from local context and a reasonable assumption would be risky, ask the user directly with a concise plain-text question. Never write a multiple choice question as a textual assistant message. ${browserToolInstructions(browserToolsAvailable)} ${T3_CODE_THREAD_REFERENCE_INSTRUCTIONS} +${T3_CODE_COMPUTER_TOOL_INSTRUCTIONS} `; export interface CodexRuntimeInfo { diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts index 6a2ea9e8d308..0b8260067d20 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts @@ -452,6 +452,8 @@ describe("buildCodexDeveloperInstructions", () => { NodeAssert.ok(instructions.startsWith(codexDefaultModeDeveloperInstructions(true))); NodeAssert.match(instructions, /T3 Code/); + NodeAssert.match(instructions, /computer_list/); + NodeAssert.match(instructions, /computer_send/); NodeAssert.match(instructions, /Codex harness/); NodeAssert.match(instructions, /as gpt-5\.3-codex with high reasoning effort/); }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index b55bd6c51028..bb449d39a48d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -52,6 +52,7 @@ import * as TerminalManager from "./terminal/Manager.ts"; import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as ComputerTaskBroker from "./mcp/ComputerTaskBroker.ts"; import * as PreviewManager from "./preview/Manager.ts"; import * as PortScanner from "./preview/PortScanner.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -516,6 +517,7 @@ export const makeRoutesLayer = Layer.mergeAll( Layer.provide(CodexPluginMarketplace.layer), Layer.provide(PullRequestStackServiceLive), Layer.provide(PreviewAutomationBroker.layer), + Layer.provide(ComputerTaskBroker.layer), Layer.provide(ServerSelfUpdate.layer), Layer.provide(commandReadinessLayer), Layer.provide(browserApiCorsLayer), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 53df9bcf68d7..7c3c3c9e4f43 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -88,6 +88,7 @@ import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; +import * as ComputerTaskBroker from "./mcp/ComputerTaskBroker.ts"; import * as PreviewManager from "./preview/Manager.ts"; import { issueAssetUrl } from "./assets/AssetAccess.ts"; import * as PortScanner from "./preview/PortScanner.ts"; @@ -362,6 +363,7 @@ function toAuthAccessStreamEvent( const makeWsRpcLayer = ( currentSession: EnvironmentAuth.AuthenticatedSession, previewAutomationBroker: PreviewAutomationBroker.PreviewAutomationBroker["Service"], + computerTaskBroker: ComputerTaskBroker.ComputerTaskBroker["Service"], ) => WsRpcGroup.toLayer( Effect.gen(function* () { @@ -2318,6 +2320,18 @@ const makeWsRpcLayer = ( previewAutomationBroker.focusHost(input), { "rpc.aggregate": "preview-automation" }, ), + [WS_METHODS.computersConnect]: (input) => + observeRpcStreamEffect(WS_METHODS.computersConnect, computerTaskBroker.connect(input), { + "rpc.aggregate": "computers", + }), + [WS_METHODS.computersSync]: (input) => + observeRpcEffect(WS_METHODS.computersSync, computerTaskBroker.sync(input), { + "rpc.aggregate": "computers", + }), + [WS_METHODS.computersRespond]: (input) => + observeRpcEffect(WS_METHODS.computersRespond, computerTaskBroker.respond(input), { + "rpc.aggregate": "computers", + }), [WS_METHODS.subscribePreviewEvents]: (_input) => observeRpcStream(WS_METHODS.subscribePreviewEvents, previewManager.events, { "rpc.aggregate": "preview", @@ -2495,6 +2509,7 @@ const makeWsRpcLayer = ( export const websocketRpcRouteLayer = Layer.unwrap( Effect.gen(function* () { const previewAutomationBroker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const computerTaskBroker = yield* ComputerTaskBroker.ComputerTaskBroker; const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const pullRequests = yield* PullRequestService.PullRequestService; const pullRequestStacks = yield* GitHubPullRequestStackService.GitHubPullRequestStackService; @@ -2517,7 +2532,7 @@ export const websocketRpcRouteLayer = Layer.unwrap( disableTracing: true, }).pipe( Effect.provide( - makeWsRpcLayer(session, previewAutomationBroker).pipe( + makeWsRpcLayer(session, previewAutomationBroker, computerTaskBroker).pipe( Layer.provideMerge(RpcSerialization.layerJson), Layer.provide(ProviderMaintenanceRunner.layer), Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)), diff --git a/apps/web/src/AppRoot.test.tsx b/apps/web/src/AppRoot.test.tsx index 791004b74fad..5675e8988678 100644 --- a/apps/web/src/AppRoot.test.tsx +++ b/apps/web/src/AppRoot.test.tsx @@ -3,6 +3,7 @@ import { RouterProvider } from "@tanstack/react-router"; import { describe, expect, it } from "vite-plus/test"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; +import { ComputerTaskHosts } from "./components/computers/ComputerTaskHosts"; import { PreviewAutomationHosts } from "./components/preview/PreviewAutomationHosts"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; @@ -17,10 +18,11 @@ describe("AppRoot", () => { const children = Children.toArray( (root as ReactElement<{ readonly children: ReactNode }>).props.children, ); - expect(children).toHaveLength(4); + expect(children).toHaveLength(5); expect(isValidElement(children[0]) && children[0].type).toBe(RouterProvider); expect(isValidElement(children[1]) && children[1].type).toBe(PreviewAutomationHosts); - expect(isValidElement(children[2]) && children[2].type).toBe(ElectronBrowserHost); - expect(isValidElement(children[3]) && children[3].type).toBe(QuitHoldOverlay); + expect(isValidElement(children[2]) && children[2].type).toBe(ComputerTaskHosts); + expect(isValidElement(children[3]) && children[3].type).toBe(ElectronBrowserHost); + expect(isValidElement(children[4]) && children[4].type).toBe(QuitHoldOverlay); }); }); diff --git a/apps/web/src/AppRoot.tsx b/apps/web/src/AppRoot.tsx index 857125c9fdaf..f9bc1bb26693 100644 --- a/apps/web/src/AppRoot.tsx +++ b/apps/web/src/AppRoot.tsx @@ -1,6 +1,7 @@ import { RouterProvider } from "@tanstack/react-router"; import { ElectronBrowserHost } from "./browser/ElectronBrowserHost"; +import { ComputerTaskHosts } from "./components/computers/ComputerTaskHosts"; import { PreviewAutomationHosts } from "./components/preview/PreviewAutomationHosts"; import { QuitHoldOverlay } from "./components/QuitHoldOverlay"; import { AppAtomRegistryProvider } from "./rpc/atomRegistry"; @@ -16,6 +17,7 @@ export function AppRoot({ router }: { readonly router: AppRouter }) { + diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 1bbb9cf34acf..8367e146432b 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -21,11 +21,13 @@ import { PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, } from "@t3tools/contracts"; import type { EnvironmentConnectionPresentation } from "@t3tools/client-runtime/connection"; +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; import { serializeComposerFileLink, serializeComposerThreadLink, } from "@t3tools/shared/composerTrigger"; import { createModelSelection, normalizeModelSlug } from "@t3tools/shared/model"; +import { appendVoiceTranscript as appendVoiceTranscriptText } from "@t3tools/shared/voiceTranscription"; import { memo, type ReactNode, @@ -119,6 +121,8 @@ import { submitComposerDraft, } from "./composerSubmission"; import { ComposerPromptLengthValidation } from "./ComposerPromptLengthValidation"; +import { useVoiceTranscription } from "../../hooks/useVoiceTranscription"; +import { VoiceTranscriptionPanel } from "./VoiceTranscriptionPanel"; type ComposerCommandMenuPosition = { bottom: number; @@ -226,6 +230,7 @@ import { LockOpenIcon, PenLineIcon, SparklesIcon, + MicIcon, XIcon, } from "lucide-react"; import { proposedPlanTitle } from "../../proposedPlan"; @@ -704,6 +709,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) onExpandImage, } = props; const isSendDisabled = sendDisabledReason !== null; + const voiceTranscriptionTargetKey = + typeof composerDraftTarget === "string" + ? composerDraftTarget + : scopedThreadKey(composerDraftTarget); // ------------------------------------------------------------------ // Store subscriptions (prompt / images / terminal contexts) @@ -1010,6 +1019,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const mobileComposerExpandInFlightRef = useRef(false); const stashPulseKeyRef = useRef(0); const stashPulseTimeoutRef = useRef(null); + const submitComposerRef = useRef<(event?: { preventDefault: () => void }) => void>(() => {}); + const voiceTranscriptionTargetKeyRef = useRef(voiceTranscriptionTargetKey); + const voiceTranscriptionOriginTargetKeyRef = useRef(null); + voiceTranscriptionTargetKeyRef.current = voiceTranscriptionTargetKey; /** * Snapshots currently being encoded, keyed by target+prompt+image ids. * Keyed rather than boolean so a genuinely different prompt (or a different @@ -1296,6 +1309,14 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) projectSelectionRequired || environmentUnavailable !== null || !composerSendState.hasSendableContent; + const voiceTranscriptionSendDisabled = + phase === "running" || + isSendBusy || + isSendDisabled || + isConnecting || + noProviderAvailable || + projectSelectionRequired || + environmentUnavailable !== null; const collapsedComposerPrimaryActionLabel = "Send message"; const showMobilePendingAnswerActions = isMobileViewport && !isComposerCollapsedMobile && pendingPrimaryAction !== null; @@ -1310,6 +1331,71 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) [composerDraftTarget, setComposerDraftPrompt], ); + const appendVoiceTranscriptToPrompt = useCallback( + (transcript: string) => { + if ( + voiceTranscriptionOriginTargetKeyRef.current === null || + voiceTranscriptionOriginTargetKeyRef.current !== voiceTranscriptionTargetKeyRef.current + ) { + return false; + } + const currentPrompt = promptRef.current; + const nextPrompt = appendVoiceTranscriptText(currentPrompt, transcript); + if (nextPrompt === currentPrompt) return false; + promptRef.current = nextPrompt; + setPrompt(nextPrompt); + const nextCursor = collapseExpandedComposerCursor(nextPrompt, nextPrompt.length); + setComposerCursor(nextCursor); + setComposerTrigger(null); + scheduleComposerFocus(); + voiceTranscriptionOriginTargetKeyRef.current = null; + return true; + }, + [promptRef, scheduleComposerFocus, setPrompt], + ); + const voiceTranscription = useVoiceTranscription({ + config: { + provider: settings.voiceTranscriptionProvider, + apiKey: settings.voiceTranscriptionApiKey, + model: settings.voiceTranscriptionModel, + }, + onTranscriptInsert: appendVoiceTranscriptToPrompt, + onTranscriptSend: (transcript) => { + if (appendVoiceTranscriptToPrompt(transcript)) submitComposerRef.current(); + }, + }); + const voiceTranscriptionReady = + settings.voiceTranscriptionEnabled && settings.voiceTranscriptionModel.trim().length > 0; + const startVoiceTranscription = useCallback(async () => { + if (phase === "running") return; + voiceTranscriptionOriginTargetKeyRef.current = voiceTranscriptionTargetKeyRef.current; + await voiceTranscription.start(); + }, [phase, voiceTranscription.start]); + const cancelVoiceTranscription = useCallback(() => { + voiceTranscriptionOriginTargetKeyRef.current = null; + voiceTranscription.cancel(); + }, [voiceTranscription.cancel]); + + useEffect(() => { + const activeTargetKey = voiceTranscriptionTargetKey; + return () => { + if (voiceTranscriptionOriginTargetKeyRef.current !== activeTargetKey) return; + voiceTranscriptionOriginTargetKeyRef.current = null; + voiceTranscription.cancel(); + }; + }, [voiceTranscription.cancel, voiceTranscriptionTargetKey]); + + useEffect(() => { + if (voiceTranscription.status === "idle") return; + const cancelOnEscape = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + event.preventDefault(); + cancelVoiceTranscription(); + }; + window.addEventListener("keydown", cancelOnEscape); + return () => window.removeEventListener("keydown", cancelOnEscape); + }, [cancelVoiceTranscription, voiceTranscription.status]); + const addComposerImage = useCallback( (image: ComposerImageAttachment) => { addComposerDraftImage(composerDraftTarget, image); @@ -1905,7 +1991,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) showPlanFollowUpPrompt, ]); - const submitComposer = useCallback( + const submitComposerAfterTranscription = useCallback( (event?: { preventDefault: () => void }) => { if (noProviderAvailable || isSendDisabled) { event?.preventDefault(); @@ -1953,6 +2039,38 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) shouldBlurMobileComposerOnSubmit, ], ); + const submitVoiceTranscript = useCallback( + (event?: { preventDefault: () => void }) => { + if (voiceTranscriptionSendDisabled) { + event?.preventDefault(); + return; + } + submitComposerAfterTranscription(event); + }, + [submitComposerAfterTranscription, voiceTranscriptionSendDisabled], + ); + const submitComposer = useCallback( + (event?: { preventDefault: () => void }) => { + if (voiceTranscription.status === "recording") { + event?.preventDefault(); + if (voiceTranscriptionSendDisabled) return; + voiceTranscription.stop("send"); + return; + } + if (voiceTranscription.status === "transcribing") { + event?.preventDefault(); + return; + } + submitComposerAfterTranscription(event); + }, + [ + submitComposerAfterTranscription, + voiceTranscription.status, + voiceTranscription.stop, + voiceTranscriptionSendDisabled, + ], + ); + submitComposerRef.current = submitVoiceTranscript; const expandMobileComposer = useCallback(() => { if (composerBlurFrameRef.current !== null) { window.cancelAnimationFrame(composerBlurFrameRef.current); @@ -3220,8 +3338,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) message={providerInputSubmissionError ?? composerSubmissionError} /> + {voiceTranscription.status === "idle" && + voiceTranscriptionReady && + voiceTranscription.error ? ( +

+ {voiceTranscription.error} +

+ ) : null} + {/* Bottom toolbar */} - {isComposerCollapsedMobile || isComposerApprovalState ? null : ( + {(isComposerCollapsedMobile && voiceTranscription.status === "idle") || + isComposerApprovalState ? null : (
-
+ {voiceTranscription.status !== "idle" ? ( + voiceTranscription.stop("insert")} + onSend={() => voiceTranscription.stop("send")} + /> + ) : null} +
{noProviderAvailable ? ( + } + /> + Start dictation + + ) : null} + 0} + isSendBusy={isSendBusy} + sendDisabledReason={sendDisabledReason} + isConnecting={isConnecting} + isEnvironmentUnavailable={ + environmentUnavailable !== null || + noProviderAvailable || + projectSelectionRequired + } + isPreparingWorktree={isPreparingWorktree} + hasSendableContent={composerSendState.hasSendableContent} + preserveComposerFocusOnPointerDown={isMobileViewport} + showSendWhileRunning={isMobileViewport} + onPreviousPendingQuestion={onPreviousActivePendingUserInputQuestion} + onInterrupt={handleInterruptPrimaryAction} + onImplementPlanInNewThread={handleImplementPlanInNewThreadPrimaryAction} + /> +
+ ) : null}
)}
diff --git a/apps/web/src/components/chat/VoiceTranscriptionPanel.tsx b/apps/web/src/components/chat/VoiceTranscriptionPanel.tsx index 43d39185a2c3..4a53fc66e1ee 100644 --- a/apps/web/src/components/chat/VoiceTranscriptionPanel.tsx +++ b/apps/web/src/components/chat/VoiceTranscriptionPanel.tsx @@ -1,21 +1,40 @@ -import { LoaderCircleIcon } from "lucide-react"; +import type { PointerEventHandler } from "react"; +import { ArrowUpIcon, LoaderCircleIcon, SquareIcon, XIcon } from "lucide-react"; import type { VoiceTranscriptionStatus } from "../../hooks/useVoiceTranscription"; +import { Button } from "../ui/button"; function formatElapsed(elapsedMs: number): string { const seconds = Math.floor(elapsedMs / 1_000); return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; } +const preventPointerFocus: PointerEventHandler = (event) => { + event.preventDefault(); +}; + export function VoiceTranscriptionPanel({ status, elapsedMs, levels, + sendDisabled, + preserveComposerFocusOnPointerDown, + onCancel, + onStop, + onSend, }: { readonly status: Exclude; readonly elapsedMs: number; readonly levels: readonly number[]; + readonly sendDisabled: boolean; + readonly preserveComposerFocusOnPointerDown?: boolean; + readonly onCancel: () => void; + readonly onStop: () => void; + readonly onSend: () => void; }) { + const pointerFocusProps = preserveComposerFocusOnPointerDown + ? { onPointerDown: preventPointerFocus } + : undefined; const waveformPath = levels .map((level, index) => { if (level <= 0.01) return ""; @@ -24,48 +43,98 @@ export function VoiceTranscriptionPanel({ }) .join(" "); + if (status === "transcribing") { + return ( +
+ + + Processing recording… +
+ ); + } + return (
- {status === "recording" ? ( - - ) : ( -
- - Transcribing… -
- )} - + + + {formatElapsed(elapsedMs)} + +
); } diff --git a/apps/web/src/components/computers/ComputerTaskHosts.tsx b/apps/web/src/components/computers/ComputerTaskHosts.tsx new file mode 100644 index 000000000000..f4a42b0c7db6 --- /dev/null +++ b/apps/web/src/components/computers/ComputerTaskHosts.tsx @@ -0,0 +1,240 @@ +"use client"; + +import { RegistryContext, useAtomValue } from "@effect/atom-react"; +import { + computerPeerFromPresentation, + ComputerTaskDispatchError, + matchProject, + requireMatchedProject, +} from "@t3tools/client-runtime/computers"; +import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { + type ComputerPeer, + type ComputerTaskHost as ComputerTaskHostPayload, + type ComputerTaskSendRequest, + type ComputerTaskSendResult, + type ComputerTaskStreamEvent, + type EnvironmentId, + type OrchestrationShellSnapshot, +} from "@t3tools/contracts"; +import { AsyncResult, Atom, AtomRegistry } from "effect/unstable/reactivity"; +import { useCallback, useContext, useEffect, useMemo, useState } from "react"; + +import { environmentCatalog } from "~/connection/catalog"; +import { newMessageId, newThreadId } from "~/lib/utils"; +import { computerEnvironment } from "~/state/computers"; +import { useEnvironments } from "~/state/environments"; +import { environmentPresentations } from "~/state/presentation"; +import { environmentSnapshotAtom } from "~/state/shell"; +import { threadEnvironment } from "~/state/threads"; +import { useAtomCommand } from "~/state/use-atom-command"; + +const TARGET_READY_TIMEOUT_MS = 20_000; +const TARGET_READY_POLL_MS = 200; + +function createComputerTaskClientId(): string { + const bytes = globalThis.crypto.getRandomValues(new Uint8Array(16)); + return `computers-${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +type RequestStreamResult = AsyncResult.AsyncResult; + +function createComputerTaskRequestConsumerAtom(options: { + readonly requestsAtom: Atom.Atom>; + readonly clientId: ComputerTaskHostPayload["clientId"]; + readonly handle: (request: ComputerTaskSendRequest) => Promise; + readonly respond: (response: { + readonly clientId: ComputerTaskHostPayload["clientId"]; + readonly connectionId: string; + readonly requestId: string; + readonly ok: boolean; + readonly result?: ComputerTaskSendResult; + readonly error?: { + readonly code: + | "dispatch_failed" + | "project_not_found" + | "project_ambiguous" + | "computer_offline"; + readonly detail: string; + }; + }) => Promise; +}): Atom.Atom { + return Atom.make((get) => { + const consume = (result: RequestStreamResult) => { + if (!AsyncResult.isSuccess(result)) return; + const event = result.value; + if (event.type === "connected") return; + void options.handle(event.request).then( + (value) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: true, + result: value, + }), + (error) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: false, + error: { + code: error instanceof ComputerTaskDispatchError ? error.code : "dispatch_failed", + detail: + error instanceof Error + ? error.message + : "Could not start the task on that computer.", + }, + }), + ); + }; + const initial = get.once(options.requestsAtom); + get.subscribe(options.requestsAtom, consume); + queueMicrotask(() => consume(initial)); + }).pipe(Atom.withLabel(`computer-task-consumer:${options.clientId}`)); +} + +export function ComputerTaskHosts() { + const { environments } = useEnvironments(); + const computers = useMemo( + () => environments.map((environment) => computerPeerFromPresentation(environment)), + [environments], + ); + if (environments.length === 0) return null; + return ( + <> + {environments.map((environment) => ( + + ))} + + ); +} + +function ComputerTaskHost(props: { + readonly environmentId: EnvironmentId; + readonly computers: ReadonlyArray; +}) { + const { environmentId, computers } = props; + const registry = useContext(RegistryContext); + const [clientId] = useState(createComputerTaskClientId); + const [connectHost] = useState(() => ({ clientId, computers })); + const requestsAtom = computerEnvironment.requests({ environmentId, input: connectHost }); + const respond = useAtomCommand(computerEnvironment.respond, { reportFailure: false }); + const sync = useAtomCommand(computerEnvironment.sync, { reportFailure: false }); + const startTurn = useAtomCommand(threadEnvironment.startTurn, { reportFailure: false }); + const retryNow = useAtomCommand(environmentCatalog.retryNow, { reportFailure: false }); + const thisPresentation = useAtomValue(environmentPresentations.presentationAtom(environmentId)); + useAtomValue(environmentSnapshotAtom(environmentId)); + + useEffect(() => { + if (thisPresentation?.connection.phase !== "connected") return; + void sync({ environmentId, input: { clientId, computers } }); + }, [clientId, computers, environmentId, sync, thisPresentation?.connection.phase]); + + const handleRequest = useCallback( + async (request: ComputerTaskSendRequest): Promise => { + const targetId = request.computer.environmentId; + const livePresentation = registry + .get(environmentPresentations.presentationsAtom) + .get(targetId); + if (livePresentation && livePresentation.connection.phase !== "connected") { + await retryNow(targetId); + } + const snapshot = await waitForTargetSnapshot(registry, targetId); + const matched = requireMatchedProject( + matchProject({ + projects: snapshot.projects, + hint: request.projectHint, + sourceTitle: request.source.projectTitle, + sourceWorkspaceRoot: request.source.projectWorkspaceRoot, + }), + ); + const threadId = newThreadId(); + const messageId = newMessageId(); + const createdAt = new Date().toISOString(); + const startResult = await startTurn({ + environmentId: targetId, + input: { + threadId, + message: { + messageId, + role: "user", + text: request.message, + attachments: [], + }, + modelSelection: request.modelSelection, + titleSeed: request.title, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + bootstrap: { + createThread: { + projectId: matched.id, + title: request.title, + modelSelection: request.modelSelection, + runtimeMode: request.runtimeMode, + interactionMode: request.interactionMode, + branch: null, + worktreePath: null, + createdAt, + }, + }, + createdAt, + }, + }); + if (startResult._tag === "Failure") { + throw squashAtomCommandFailure(startResult); + } + return { + environmentId: targetId, + threadId, + projectId: matched.id, + }; + }, + [registry, retryNow, startTurn], + ); + + const consumerAtom = useMemo( + () => + createComputerTaskRequestConsumerAtom({ + requestsAtom, + clientId, + handle: handleRequest, + respond: (response) => respond({ environmentId, input: response }), + }), + [clientId, environmentId, handleRequest, requestsAtom, respond], + ); + useAtomValue(consumerAtom); + return null; +} + +async function waitForTargetSnapshot( + registry: AtomRegistry.AtomRegistry, + targetId: EnvironmentId, +): Promise { + const deadline = Date.now() + TARGET_READY_TIMEOUT_MS; + let lastPhase: string | null = null; + while (Date.now() < deadline) { + const presentation = registry.get(environmentPresentations.presentationsAtom).get(targetId); + lastPhase = presentation?.connection.phase ?? null; + const snapshot = registry.get(environmentSnapshotAtom(targetId)); + if (lastPhase === "connected" && snapshot !== null) { + return snapshot; + } + await sleep(TARGET_READY_POLL_MS); + } + throw new ComputerTaskDispatchError( + "computer_offline", + lastPhase === "connected" + ? "That computer connected but T3 has not loaded its projects yet." + : "Could not reach that computer. Keep T3 Code running there, or pick it in Run on first.", + ); +} diff --git a/apps/web/src/components/settings/SettingsPanels.logic.test.ts b/apps/web/src/components/settings/SettingsPanels.logic.test.ts index 5c715eb4eb25..17dbc795dd25 100644 --- a/apps/web/src/components/settings/SettingsPanels.logic.test.ts +++ b/apps/web/src/components/settings/SettingsPanels.logic.test.ts @@ -16,11 +16,58 @@ import { getChangedTypographySettingLabels, isSamePreviewViewport, hasChangedBackgroundActivitySettings, + hasChangedVoiceTranscriptionSettings, isProjectGroupingEnabled, projectGroupingModeFromToggle, resolveBackgroundActivityProfileOption, + shouldRestoreVoiceTranscriptionDefaults, + voiceTranscriptionModelOptions, } from "./SettingsPanels.logic"; +describe("voice transcription settings", () => { + it("only restores voice settings that were included in the confirmed reset", () => { + const liveSettings = { + ...DEFAULT_UNIFIED_SETTINGS, + voiceTranscriptionApiKey: "key-saved-from-another-tab", + voiceTranscriptionModel: "gpt-4o-transcribe", + voiceTranscriptionEnabled: true, + }; + + expect(hasChangedVoiceTranscriptionSettings(DEFAULT_UNIFIED_SETTINGS)).toBe(false); + expect(hasChangedVoiceTranscriptionSettings(liveSettings)).toBe(true); + expect( + shouldRestoreVoiceTranscriptionDefaults({ + wasIncludedInConfirmation: false, + liveSettings, + }), + ).toBe(false); + expect( + shouldRestoreVoiceTranscriptionDefaults({ + wasIncludedInConfirmation: true, + liveSettings, + }), + ).toBe(true); + expect( + shouldRestoreVoiceTranscriptionDefaults({ + wasIncludedInConfirmation: true, + liveSettings: DEFAULT_UNIFIED_SETTINGS, + }), + ).toBe(false); + }); + + it("keeps a selected model even when the latest provider list omits it", () => { + expect(voiceTranscriptionModelOptions(["gpt-4o-mini-transcribe"], "gpt-4o-transcribe")).toEqual( + ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"], + ); + expect( + voiceTranscriptionModelOptions( + ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"], + "gpt-4o-transcribe", + ), + ).toEqual(["gpt-4o-transcribe", "gpt-4o-mini-transcribe"]); + }); +}); + describe("typography settings restore", () => { it("detects family and size changes by font row", () => { expect(getChangedTypographySettingLabels(DEFAULT_UNIFIED_SETTINGS)).toEqual([]); diff --git a/apps/web/src/components/settings/SettingsPanels.logic.ts b/apps/web/src/components/settings/SettingsPanels.logic.ts index a5d5d9958498..77afa7670442 100644 --- a/apps/web/src/components/settings/SettingsPanels.logic.ts +++ b/apps/web/src/components/settings/SettingsPanels.logic.ts @@ -87,6 +87,43 @@ type TypographySettings = Pick< | "fontSizeTerminal" >; +type VoiceTranscriptionSettings = Pick< + UnifiedSettings, + | "voiceTranscriptionEnabled" + | "voiceTranscriptionProvider" + | "voiceTranscriptionApiKey" + | "voiceTranscriptionModel" +>; + +export function hasChangedVoiceTranscriptionSettings( + settings: VoiceTranscriptionSettings, +): boolean { + return ( + settings.voiceTranscriptionEnabled !== DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionEnabled || + settings.voiceTranscriptionProvider !== DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionProvider || + settings.voiceTranscriptionApiKey !== DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionApiKey || + settings.voiceTranscriptionModel !== DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionModel + ); +} + +export function shouldRestoreVoiceTranscriptionDefaults(input: { + readonly wasIncludedInConfirmation: boolean; + readonly liveSettings: VoiceTranscriptionSettings; +}): boolean { + return ( + input.wasIncludedInConfirmation && hasChangedVoiceTranscriptionSettings(input.liveSettings) + ); +} + +export function voiceTranscriptionModelOptions( + availableModels: readonly string[], + selectedModel: string, +): readonly string[] { + return selectedModel && !availableModels.includes(selectedModel) + ? [selectedModel, ...availableModels] + : availableModels; +} + /** Labels the font rows whose family or size differs from the defaults. */ export function getChangedTypographySettingLabels(settings: TypographySettings): string[] { return [ diff --git a/apps/web/src/components/settings/SettingsPanels.tsx b/apps/web/src/components/settings/SettingsPanels.tsx index 09de41e61720..620669e5cd0f 100644 --- a/apps/web/src/components/settings/SettingsPanels.tsx +++ b/apps/web/src/components/settings/SettingsPanels.tsx @@ -64,7 +64,11 @@ import { useTheme, } from "../../hooks/useTheme"; import { useLocalStorage } from "../../hooks/useLocalStorage"; -import { usePrimarySettings, useUpdatePrimarySettings } from "../../hooks/useSettings"; +import { + getClientSettings, + usePrimarySettings, + useUpdatePrimarySettings, +} from "../../hooks/useSettings"; import { useThreadActions } from "../../hooks/useThreadActions"; import { useDesktopUpdateState } from "../../state/desktopUpdate"; import { @@ -82,10 +86,6 @@ import { primaryServerObservabilityAtom, primaryServerProvidersAtom } from "../. import { useProjects } from "../../state/entities"; import { useArchivedThreadSnapshots } from "../../lib/archivedThreadsState"; import { formatRelativeTimeLabel } from "../../timestampFormat"; -import { - listVoiceTranscriptionModels, - readVoiceTranscriptionEnvironmentStatus, -} from "../../lib/voiceTranscription"; import { Button } from "../ui/button"; import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "../ui/collapsible"; import { @@ -130,6 +130,7 @@ import { formatDiagnosticsDescription, getChangedBrowserSettingLabels, getChangedTypographySettingLabels, + hasChangedVoiceTranscriptionSettings, normalizeIntervalSeconds, PROVIDER_HEALTH_INTERVAL_STEP_SECONDS, hasChangedBackgroundActivitySettings, @@ -138,6 +139,8 @@ import { readLastEnabledProjectGroupingMode, rememberEnabledProjectGroupingMode, resolveBackgroundActivityProfileOption, + shouldRestoreVoiceTranscriptionDefaults, + voiceTranscriptionModelOptions, } from "./SettingsPanels.logic"; import { PolicyTooltip, @@ -150,6 +153,10 @@ import { import { searchableSetting } from "./settingsSearch"; import { ProjectFavicon } from "../ProjectFavicon"; import { DesktopNotificationsSettings } from "./DesktopNotificationsSettings"; +import { + listVoiceTranscriptionModels, + readVoiceTranscriptionEnvironmentStatus, +} from "../../lib/voiceTranscription"; const ENVIRONMENT_IDENTIFICATION_LABELS: Record = { artwork: "Artwork", @@ -490,6 +497,7 @@ export function useSettingsRestore(onRestored?: () => void) { DEFAULT_UNIFIED_SETTINGS.desktopNotifications.events.completion || settings.desktopNotifications.events.failure !== DEFAULT_UNIFIED_SETTINGS.desktopNotifications.events.failure; + const isVoiceTranscriptionDirty = hasChangedVoiceTranscriptionSettings(settings); const changedSettingLabels = useMemo( () => [ @@ -557,6 +565,7 @@ export function useSettingsRestore(onRestored?: () => void) { ...(settings.enableAgentBrowserAccess !== DEFAULT_UNIFIED_SETTINGS.enableAgentBrowserAccess ? ["Agent browser access"] : []), + ...(isVoiceTranscriptionDirty ? ["Voice dictation"] : []), ], [ isTextGenerationModelDirty, @@ -568,6 +577,7 @@ export function useSettingsRestore(onRestored?: () => void) { settings.browserAutoShowFloatingPreview, settings.enableAgentBrowserAccess, settings.confirmQuit, + isVoiceTranscriptionDirty, settings.confirmThreadArchive, settings.confirmThreadDelete, settings.addProjectBaseDirectory, @@ -609,6 +619,11 @@ export function useSettingsRestore(onRestored?: () => void) { ); if (!confirmed) return; + const shouldResetVoiceTranscription = shouldRestoreVoiceTranscriptionDefaults({ + wasIncludedInConfirmation: isVoiceTranscriptionDirty, + liveSettings: getClientSettings(), + }); + // Only touch the theme keys that are actually dirty, so a theme-storage // failure cannot block restoring unrelated settings. Preferences are // re-read after the confirmation dialog: they may have changed (another @@ -684,6 +699,14 @@ export function useSettingsRestore(onRestored?: () => void) { confirmThreadDelete: DEFAULT_UNIFIED_SETTINGS.confirmThreadDelete, confirmQuit: DEFAULT_UNIFIED_SETTINGS.confirmQuit, textGenerationModelSelection: DEFAULT_UNIFIED_SETTINGS.textGenerationModelSelection, + ...(shouldResetVoiceTranscription + ? { + voiceTranscriptionEnabled: DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionEnabled, + voiceTranscriptionProvider: DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionProvider, + voiceTranscriptionApiKey: DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionApiKey, + voiceTranscriptionModel: DEFAULT_UNIFIED_SETTINGS.voiceTranscriptionModel, + } + : {}), fontFamilySans: DEFAULT_UNIFIED_SETTINGS.fontFamilySans, fontFamilyComposer: DEFAULT_UNIFIED_SETTINGS.fontFamilyComposer, fontFamilyCode: DEFAULT_UNIFIED_SETTINGS.fontFamilyCode, @@ -705,6 +728,7 @@ export function useSettingsRestore(onRestored?: () => void) { }, [ changedSettingLabels, clearThemeHalves, + isVoiceTranscriptionDirty, onRestored, setFollowSystem, setTheme, @@ -1918,6 +1942,207 @@ function AutoSettleDaysInput({ ); } +function VoiceDictationSettingsSection() { + const settings = usePrimarySettings(); + const updateSettings = useUpdatePrimarySettings(); + const [environmentApiKeys, setEnvironmentApiKeys] = useState({ openai: false, groq: false }); + const [environmentStatusLoading, setEnvironmentStatusLoading] = useState(true); + const [environmentStatusError, setEnvironmentStatusError] = useState(null); + const [environmentStatusAttempt, setEnvironmentStatusAttempt] = useState(0); + const [models, setModels] = useState([]); + const [modelsLoading, setModelsLoading] = useState(false); + const [modelsError, setModelsError] = useState(null); + const provider = settings.voiceTranscriptionProvider; + const apiKey = settings.voiceTranscriptionApiKey; + const model = settings.voiceTranscriptionModel; + const selectableModels = voiceTranscriptionModelOptions(models, model); + + useEffect(() => { + let active = true; + setEnvironmentStatusLoading(true); + setEnvironmentStatusError(null); + void readVoiceTranscriptionEnvironmentStatus() + .then((status) => { + if (!active) return; + setEnvironmentApiKeys(status); + setEnvironmentStatusLoading(false); + }) + .catch((cause: unknown) => { + if (!active) return; + setEnvironmentStatusLoading(false); + setEnvironmentStatusError( + cause instanceof Error ? cause.message : "Could not check server transcription keys.", + ); + }); + return () => { + active = false; + }; + }, [environmentStatusAttempt]); + + const providerLabel = provider === "openai" ? "OpenAI" : "Groq"; + const environmentVariable = TRANSCRIPTION_API_KEY_ENV[provider]; + const hasEnvironmentApiKey = environmentApiKeys[provider]; + const hasApiKey = apiKey.trim().length > 0 || hasEnvironmentApiKey; + + useEffect(() => { + if (!hasApiKey) { + setModels([]); + setModelsLoading(false); + setModelsError(null); + return; + } + + let active = true; + setModelsLoading(true); + setModelsError(null); + const timeout = window.setTimeout( + () => { + void listVoiceTranscriptionModels({ provider, apiKey }) + .then((nextModels) => { + if (!active) return; + setModels(nextModels); + setModelsLoading(false); + }) + .catch((cause: unknown) => { + if (!active) return; + setModels([]); + setModelsLoading(false); + setModelsError( + cause instanceof Error ? cause.message : "Could not load transcription models.", + ); + }); + }, + apiKey.trim() ? 400 : 0, + ); + + return () => { + active = false; + window.clearTimeout(timeout); + }; + }, [apiKey, hasApiKey, provider]); + + const modelDescription = !hasApiKey + ? environmentStatusLoading + ? `Checking the connected server for ${environmentVariable}…` + : environmentStatusError + ? `${environmentStatusError} Add a client key or retry the server check.` + : `Save an API key to load ${providerLabel} transcription models.` + : modelsLoading + ? `Loading models available from ${providerLabel}…` + : modelsError + ? modelsError + : models.length === 0 + ? `${providerLabel} did not return any models.` + : model && !models.includes(model) + ? "The saved model was not returned by the provider. Keep it or choose another model." + : "Choose a model. The microphone appears in the composer after that."; + + return ( + + + updateSettings({ + voiceTranscriptionProvider: value as VoiceTranscriptionProvider, + voiceTranscriptionApiKey: "", + voiceTranscriptionModel: "", + voiceTranscriptionEnabled: false, + }) + } + > + + {providerLabel} + + + + OpenAI + + + Groq + + + + } + /> + + + updateSettings({ + voiceTranscriptionApiKey: event.target.value, + voiceTranscriptionModel: "", + voiceTranscriptionEnabled: false, + }) + } + placeholder={hasEnvironmentApiKey ? `Using ${environmentVariable}` : "Required"} + aria-label={`${providerLabel} transcription API key`} + /> + {environmentStatusError ? ( + + ) : null} + + } + /> + { + if (value !== null) { + updateSettings({ + voiceTranscriptionModel: value, + voiceTranscriptionEnabled: true, + }); + } + }} + > + + + {model || (modelsLoading ? "Loading models…" : "Select model")} + + + + {selectableModels.map((availableModel) => ( + + {availableModel} + + ))} + + + } + /> + + ); +} + // The legacy rows sit behind the fold, so a settings-search jump has to // expand the section before its target can mount and scroll. const LEGACY_FEATURE_TARGET_IDS: ReadonlySet = new Set([ @@ -2617,6 +2842,7 @@ export function GeneralSettingsPanel() { + {isElectron || HOSTED_APP_CHANNEL ? ( diff --git a/apps/web/src/environments/primary/httpLayer.ts b/apps/web/src/environments/primary/httpLayer.ts index 306b5de63d75..31bbc2d52748 100644 --- a/apps/web/src/environments/primary/httpLayer.ts +++ b/apps/web/src/environments/primary/httpLayer.ts @@ -33,7 +33,7 @@ function withPrimaryBearerToken(client: HttpClient.HttpClient): HttpClient.HttpC export function makePrimaryEnvironmentHttpLayer() { return Layer.unwrap( Effect.sync(() => { - const baseLayer = remoteHttpClientLayer(globalThis.fetch); + const baseLayer = remoteHttpClientLayer((input, init) => globalThis.fetch(input, init)); if (isSameOriginBrowserPrimary()) { return Layer.merge( baseLayer, diff --git a/apps/web/src/hooks/useVoiceTranscription.ts b/apps/web/src/hooks/useVoiceTranscription.ts index 22e776dafc7a..28caae793c84 100644 --- a/apps/web/src/hooks/useVoiceTranscription.ts +++ b/apps/web/src/hooks/useVoiceTranscription.ts @@ -1,8 +1,13 @@ +import { + resolveVoiceTranscriptionAction, + type VoiceTranscriptionAction, +} from "@t3tools/shared/voiceTranscription"; import { useCallback, useEffect, useRef, useState } from "react"; import { transcribeVoiceRecording, type VoiceTranscriptionConfig } from "../lib/voiceTranscription"; const LEVEL_COUNT = 160; +const MIN_RECORDING_MS = 250; const MAX_RECORDING_MS = 5 * 60 * 1_000; const MIME_TYPES = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/mp4"]; const FLAT_LEVELS = Array(LEVEL_COUNT).fill(0); @@ -16,28 +21,38 @@ function supportedMimeType(): string | undefined { export function useVoiceTranscription({ config, - onTranscript, + onTranscriptInsert, + onTranscriptSend, }: { readonly config: VoiceTranscriptionConfig; - readonly onTranscript: (text: string) => void; + readonly onTranscriptInsert: (text: string) => void; + readonly onTranscriptSend: (text: string) => void; }) { const [status, setStatus] = useState("idle"); const [elapsedMs, setElapsedMs] = useState(0); const [levels, setLevels] = useState(FLAT_LEVELS); const [error, setError] = useState(null); const recorderRef = useRef(null); + const statusRef = useRef(status); const startingRef = useRef(false); const cancelStartingRef = useRef(false); + const restartAfterCancellationRef = useRef(false); const streamRef = useRef(null); const audioContextRef = useRef(null); const intervalsRef = useRef([]); const timeoutRef = useRef(null); const startedAtRef = useRef(0); + const transcriptionAttemptRef = useRef(0); + const terminalActionRef = useRef(null); const mountedRef = useRef(true); const configRef = useRef(config); - const onTranscriptRef = useRef(onTranscript); + const onTranscriptInsertRef = useRef(onTranscriptInsert); + const onTranscriptSendRef = useRef(onTranscriptSend); + const startRef = useRef<() => Promise>(async () => undefined); + statusRef.current = status; configRef.current = config; - onTranscriptRef.current = onTranscript; + onTranscriptInsertRef.current = onTranscriptInsert; + onTranscriptSendRef.current = onTranscriptSend; const cleanupCapture = useCallback(() => { for (const interval of intervalsRef.current) window.clearInterval(interval); @@ -57,29 +72,64 @@ export function useVoiceTranscription({ mountedRef.current = false; startingRef.current = false; cancelStartingRef.current = true; + restartAfterCancellationRef.current = false; + transcriptionAttemptRef.current += 1; + terminalActionRef.current = "abort"; const recorder = recorderRef.current; if (recorder?.state === "recording") recorder.stop(); cleanupCapture(); }; }, [cleanupCapture]); - const stop = useCallback(() => { + const stop = useCallback( + (action: Exclude = "insert") => { + terminalActionRef.current = resolveVoiceTranscriptionAction( + terminalActionRef.current, + action, + ); + if (startingRef.current) { + // Permission acquisition is already in flight. Preserve the requested + // action and apply it as soon as MediaRecorder starts. + return; + } + const recorder = recorderRef.current; + if (recorder?.state === "recording") recorder.stop(); + }, + [cleanupCapture], + ); + + const cancel = useCallback(() => { + terminalActionRef.current = "abort"; if (startingRef.current) { cancelStartingRef.current = true; cleanupCapture(); if (mountedRef.current) { + statusRef.current = "idle"; setStatus("idle"); setElapsedMs(0); } return; } + if (statusRef.current === "transcribing") { + transcriptionAttemptRef.current += 1; + terminalActionRef.current = null; + statusRef.current = "idle"; + setStatus("idle"); + setElapsedMs(0); + return; + } const recorder = recorderRef.current; if (recorder?.state === "recording") recorder.stop(); }, [cleanupCapture]); const start = useCallback(async () => { - if (startingRef.current || status !== "idle") return; + if (startingRef.current) { + if (cancelStartingRef.current) restartAfterCancellationRef.current = true; + return; + } + if (statusRef.current !== "idle") return; startingRef.current = true; + terminalActionRef.current = null; setError(null); setElapsedMs(0); setLevels(FLAT_LEVELS); @@ -95,6 +145,8 @@ export function useVoiceTranscription({ } cancelStartingRef.current = false; + restartAfterCancellationRef.current = false; + statusRef.current = "recording"; setStatus("recording"); try { const audioContext = new AudioContext(); @@ -109,6 +161,12 @@ export function useVoiceTranscription({ stream.getTracks().forEach((track) => track.stop()); startingRef.current = false; cancelStartingRef.current = false; + terminalActionRef.current = null; + const shouldRestart = restartAfterCancellationRef.current; + restartAfterCancellationRef.current = false; + if (shouldRestart && mountedRef.current) { + queueMicrotask(() => void startRef.current()); + } return; } @@ -124,26 +182,51 @@ export function useVoiceTranscription({ recorder.addEventListener("error", () => { recordingFailed = true; cleanupCapture(); + terminalActionRef.current = null; if (mountedRef.current) { setStatus("idle"); setError("The microphone stopped unexpectedly."); } }); recorder.addEventListener("stop", () => { + const durationMs = Date.now() - startedAtRef.current; + const requestedAction = terminalActionRef.current ?? "insert"; const blob = new Blob(chunks, { type: recorder.mimeType || mimeType || "audio/webm" }); cleanupCapture(); if (!mountedRef.current || recordingFailed) return; + if (requestedAction === "abort" || durationMs < MIN_RECORDING_MS || blob.size === 0) { + terminalActionRef.current = null; + statusRef.current = "idle"; + setStatus("idle"); + setElapsedMs(0); + return; + } + + const transcriptionAttempt = ++transcriptionAttemptRef.current; + statusRef.current = "transcribing"; setStatus("transcribing"); void transcribeVoiceRecording(blob, configRef.current) .then((text) => { - if (!mountedRef.current) return; - if (text) onTranscriptRef.current(text); + if (!mountedRef.current || transcriptionAttempt !== transcriptionAttemptRef.current) { + return; + } + const finalAction = terminalActionRef.current ?? requestedAction; + if (text && finalAction !== "abort") { + if (finalAction === "send") onTranscriptSendRef.current(text); + else onTranscriptInsertRef.current(text); + } + terminalActionRef.current = null; + statusRef.current = "idle"; setStatus("idle"); setElapsedMs(0); }) .catch((cause: unknown) => { - if (!mountedRef.current) return; + if (!mountedRef.current || transcriptionAttempt !== transcriptionAttemptRef.current) { + return; + } + terminalActionRef.current = null; setError(cause instanceof Error ? cause.message : "Voice transcription failed."); + statusRef.current = "idle"; setStatus("idle"); }); }); @@ -176,16 +259,33 @@ export function useVoiceTranscription({ intervalsRef.current.push( window.setInterval(() => setElapsedMs(Date.now() - startedAtRef.current), 250), ); - timeoutRef.current = window.setTimeout(() => recorder.stop(), MAX_RECORDING_MS); + timeoutRef.current = window.setTimeout(() => { + terminalActionRef.current = terminalActionRef.current ?? "insert"; + if (recorder.state === "recording") recorder.stop(); + }, MAX_RECORDING_MS); recorder.start(250); startingRef.current = false; + const pendingAction = terminalActionRef.current; + if ( + (pendingAction === "insert" || pendingAction === "send") && + recorder.state === "recording" + ) { + recorder.stop(); + } } catch (cause) { startingRef.current = false; cleanupCapture(); + terminalActionRef.current = null; if (cancelStartingRef.current) { cancelStartingRef.current = false; + const shouldRestart = restartAfterCancellationRef.current; + restartAfterCancellationRef.current = false; + if (shouldRestart && mountedRef.current) { + queueMicrotask(() => void startRef.current()); + } return; } + statusRef.current = "idle"; setStatus("idle"); setError( cause instanceof DOMException && cause.name === "NotAllowedError" @@ -193,7 +293,8 @@ export function useVoiceTranscription({ : "Could not start the microphone.", ); } - }, [cleanupCapture, status]); + }, [cleanupCapture]); + startRef.current = start; - return { status, elapsedMs, levels, error, start, stop } as const; + return { status, elapsedMs, levels, error, start, stop, cancel } as const; } diff --git a/apps/web/src/lib/runtime.ts b/apps/web/src/lib/runtime.ts index 3836d2a39169..48d055a8d331 100644 --- a/apps/web/src/lib/runtime.ts +++ b/apps/web/src/lib/runtime.ts @@ -1,6 +1,7 @@ import * as ManagedRuntime from "effect/ManagedRuntime"; import type * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as HttpClient from "effect/unstable/http/HttpClient"; import * as Socket from "effect/unstable/socket/Socket"; import { remoteHttpClientLayer } from "@t3tools/client-runtime/rpc"; @@ -34,7 +35,10 @@ type RuntimeLayerSource = export const remoteHttpRuntime = ManagedRuntime.make(httpClientLayer); const primaryHttpRuntime = ManagedRuntime.make( - PrimaryEnvironmentHttpClient.layer.pipe(Layer.provide(primaryEnvironmentHttpLayer)), + Layer.merge( + primaryEnvironmentHttpLayer, + PrimaryEnvironmentHttpClient.layer.pipe(Layer.provide(primaryEnvironmentHttpLayer)), + ), ); export type PrimaryHttpEffectRunner = ( @@ -50,10 +54,26 @@ export const runPrimaryHttp = ( effect: Effect.Effect, ) => primaryHttpRunner(effect); +export type PrimaryRawHttpEffectRunner = ( + effect: Effect.Effect, +) => Promise; + +const livePrimaryRawHttpRunner: PrimaryRawHttpEffectRunner = (effect) => + primaryHttpRuntime.runPromise(effect); + +let primaryRawHttpRunner = livePrimaryRawHttpRunner; + +export const runPrimaryRawHttp = (effect: Effect.Effect) => + primaryRawHttpRunner(effect); + export function __setPrimaryHttpRunnerForTests(runner?: PrimaryHttpEffectRunner): void { primaryHttpRunner = runner ?? livePrimaryHttpRunner; } +export function __setPrimaryRawHttpRunnerForTests(runner?: PrimaryRawHttpEffectRunner): void { + primaryRawHttpRunner = runner ?? livePrimaryRawHttpRunner; +} + const runtimeLayer = Layer.mergeAll( httpClientLayer, browserCryptoLayer, diff --git a/apps/web/src/lib/voiceTranscription.test.ts b/apps/web/src/lib/voiceTranscription.test.ts index b82a1962f211..643a8bb44c98 100644 --- a/apps/web/src/lib/voiceTranscription.test.ts +++ b/apps/web/src/lib/voiceTranscription.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test"; import { listVoiceTranscriptionModels, + transcribeVoiceRecording, voiceTranscriptionRequestHeaders, } from "./voiceTranscription"; @@ -58,13 +59,41 @@ describe("listVoiceTranscriptionModels", () => { await expect( listVoiceTranscriptionModels({ provider: "openai", apiKey: "client-key" }), ).resolves.toEqual(["gpt-4o-mini-transcribe", "whisper-1"]); - expect(fetchMock).toHaveBeenCalledWith("http://localhost:3773/api/transcription/models", { - method: "GET", - credentials: "include", - headers: { - "x-t3-transcription-api-key": "client-key", - "x-t3-transcription-provider": "openai", + const [url, init] = fetchMock.mock.calls[0] as [URL, RequestInit]; + expect(url.toString()).toBe("http://localhost:3773/api/transcription/models"); + expect(init).toMatchObject({ method: "GET", credentials: "include" }); + expect(init.headers).toMatchObject({ + "x-t3-transcription-api-key": "client-key", + "x-t3-transcription-provider": "openai", + }); + }); + + it("uploads audio through the owned primary HTTP layer", async () => { + const fetchMock = vi.fn().mockResolvedValue(Response.json({ text: " hello " })); + vi.stubGlobal("window", { + location: { + href: "http://localhost:3773/thread", + origin: "http://localhost:3773", }, }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + transcribeVoiceRecording(new Blob(["audio"], { type: "audio/webm" }), { + provider: "openai", + apiKey: "client-key", + model: "gpt-4o-transcribe", + }), + ).resolves.toBe("hello"); + const [url, init] = fetchMock.mock.calls[0] as [URL, RequestInit]; + expect(url.toString()).toBe("http://localhost:3773/api/transcription"); + expect(init).toMatchObject({ method: "POST", credentials: "include" }); + expect(init.body).toBeInstanceOf(Uint8Array); + expect(init.headers).toMatchObject({ + "content-type": "audio/webm", + "x-t3-transcription-api-key": "client-key", + "x-t3-transcription-model": "gpt-4o-transcribe", + "x-t3-transcription-provider": "openai", + }); }); }); diff --git a/apps/web/src/lib/voiceTranscription.ts b/apps/web/src/lib/voiceTranscription.ts index 99a19cb783e6..a11dd6735238 100644 --- a/apps/web/src/lib/voiceTranscription.ts +++ b/apps/web/src/lib/voiceTranscription.ts @@ -1,7 +1,9 @@ import type { VoiceTranscriptionProvider } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { readDesktopPrimaryBearerToken } from "../environments/primary/desktopAuth"; import { resolvePrimaryEnvironmentHttpUrl } from "../environments/primary/target"; +import { runPrimaryRawHttp } from "./runtime"; export interface VoiceTranscriptionConfig { readonly provider: VoiceTranscriptionProvider; @@ -39,81 +41,91 @@ function voiceTranscriptionProviderHeaders( }; } -export async function readVoiceTranscriptionEnvironmentStatus(): Promise { - const bearerToken = await readDesktopPrimaryBearerToken(); - const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { - method: "GET", - credentials: bearerToken ? "omit" : "include", - ...(bearerToken ? { headers: { authorization: `Bearer ${bearerToken}` } } : {}), - }); - if (!response.ok) throw new Error("Could not read transcription provider settings."); +async function executeVoiceTranscriptionJsonRequest( + request: HttpClientRequest.HttpClientRequest, + failureMessage: string, +): Promise<{ readonly status: number; readonly payload: unknown }> { + try { + return await runPrimaryRawHttp( + HttpClient.execute(request).pipe( + Effect.flatMap((response) => + response.json.pipe( + Effect.orElseSucceed(() => null), + Effect.map((payload) => ({ status: response.status, payload })), + ), + ), + ), + ); + } catch (cause) { + throw new Error(failureMessage, { cause }); + } +} - const payload = (await response.json()) as { readonly openai?: unknown; readonly groq?: unknown }; +export async function readVoiceTranscriptionEnvironmentStatus(): Promise { + const { status, payload } = await executeVoiceTranscriptionJsonRequest( + HttpClientRequest.get(resolvePrimaryEnvironmentHttpUrl("/api/transcription")), + "Could not read transcription provider settings.", + ); + if (status < 200 || status >= 300) { + throw new Error("Could not read transcription provider settings."); + } + const value = payload as { readonly openai?: unknown; readonly groq?: unknown } | null; return { - openai: payload.openai === true, - groq: payload.groq === true, + openai: value?.openai === true, + groq: value?.groq === true, }; } export async function listVoiceTranscriptionModels( config: VoiceTranscriptionProviderConfig, ): Promise { - const bearerToken = await readDesktopPrimaryBearerToken(); - const response = await globalThis.fetch( + const request = HttpClientRequest.get( resolvePrimaryEnvironmentHttpUrl("/api/transcription/models"), - { - method: "GET", - credentials: bearerToken ? "omit" : "include", - headers: { - ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), - ...voiceTranscriptionProviderHeaders(config), - }, - }, + ).pipe(HttpClientRequest.setHeaders(voiceTranscriptionProviderHeaders(config))); + const { status, payload } = await executeVoiceTranscriptionJsonRequest( + request, + "Could not load transcription models.", ); - const payload = (await response.json().catch(() => null)) as { + const value = payload as { readonly models?: unknown; readonly error?: unknown; } | null; - if (!response.ok) { + if (status < 200 || status >= 300) { throw new Error( - typeof payload?.error === "string" ? payload.error : "Could not load transcription models.", + typeof value?.error === "string" ? value.error : "Could not load transcription models.", ); } - if ( - !Array.isArray(payload?.models) || - !payload.models.every((model) => typeof model === "string") - ) { + if (!Array.isArray(value?.models) || !value.models.every((model) => typeof model === "string")) { throw new Error("The transcription model response was invalid."); } - return payload.models; + return value.models; } export async function transcribeVoiceRecording( audio: Blob, config: VoiceTranscriptionConfig, ): Promise { - const bearerToken = await readDesktopPrimaryBearerToken(); - const response = await globalThis.fetch(resolvePrimaryEnvironmentHttpUrl("/api/transcription"), { - method: "POST", - credentials: bearerToken ? "omit" : "include", - headers: { - ...(bearerToken ? { authorization: `Bearer ${bearerToken}` } : {}), - ...voiceTranscriptionRequestHeaders(audio.type || "audio/webm", config), - }, - body: audio, - }); - - const payload = (await response.json().catch(() => null)) as { + const contentType = audio.type || "audio/webm"; + const audioBytes = new Uint8Array(await audio.arrayBuffer()); + const request = HttpClientRequest.post( + resolvePrimaryEnvironmentHttpUrl("/api/transcription"), + ).pipe( + HttpClientRequest.bodyUint8Array(audioBytes, contentType), + HttpClientRequest.setHeaders(voiceTranscriptionRequestHeaders(contentType, config)), + ); + const { status, payload } = await executeVoiceTranscriptionJsonRequest( + request, + "Voice transcription failed.", + ); + const value = payload as { readonly text?: unknown; readonly error?: unknown; } | null; - if (!response.ok) { - throw new Error( - typeof payload?.error === "string" ? payload.error : "Voice transcription failed.", - ); + if (status < 200 || status >= 300) { + throw new Error(typeof value?.error === "string" ? value.error : "Voice transcription failed."); } - if (typeof payload?.text !== "string") { + if (typeof value?.text !== "string") { throw new Error("The transcription response did not contain text."); } - return payload.text.trim(); + return value.text.trim(); } diff --git a/apps/web/src/rpc/requestLatencyState.ts b/apps/web/src/rpc/requestLatencyState.ts index 4ec5b56f9e2b..594e49c721e6 100644 --- a/apps/web/src/rpc/requestLatencyState.ts +++ b/apps/web/src/rpc/requestLatencyState.ts @@ -28,7 +28,10 @@ interface PendingRpcAckRequest { } const pendingRpcAckRequests = new Map(); -const untrackedRpcAckMethods = new Set([WS_METHODS.previewAutomationConnect]); +const untrackedRpcAckMethods = new Set([ + WS_METHODS.previewAutomationConnect, + WS_METHODS.computersConnect, +]); const longRunningRpcAckMethods = new Set([ WS_METHODS.serverUpdateProvider, WS_METHODS.serverRefreshProviders, diff --git a/apps/web/src/state/computers.ts b/apps/web/src/state/computers.ts new file mode 100644 index 000000000000..07c5f9c6f099 --- /dev/null +++ b/apps/web/src/state/computers.ts @@ -0,0 +1,5 @@ +import { createComputerEnvironmentAtoms } from "@t3tools/client-runtime/state/computers"; + +import { connectionAtomRuntime } from "../connection/runtime"; + +export const computerEnvironment = createComputerEnvironmentAtoms(connectionAtomRuntime); diff --git a/docs/README.md b/docs/README.md index 4455a0b758e7..f942b0fd3601 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ - [Messaging between threads](./user/thread-messaging.md) - [Review usage](./user/usage.md) - [Customize a project icon](./user/project-settings.md) +- [Voice dictation](./user/voice-dictation.md) - [Remote access](./user/remote-access.md) - [Keeping app and server in sync](./user/updating.md) - [Source control integrations](./user/source-control.md) diff --git a/docs/internals/glossary.md b/docs/internals/glossary.md index e3854b105d13..4c77194b2d99 100644 --- a/docs/internals/glossary.md +++ b/docs/internals/glossary.md @@ -17,6 +17,13 @@ This is a living glossary for T3 Code. It explains what common terms mean in thi ### Project and workspace +#### Environment + +One running T3 server and the machine it owns (filesystem, providers, terminals). Clients can be +connected to several environments at once (local, SSH, T3 Connect, paired LAN). Agents send work +to another environment with the `computer_list` / `computer_send` MCP tools; the client brokers +that hop because only it is authenticated to both servers. + #### Project The top-level workspace record in the app. In [the orchestration contracts][1], a project has a `workspaceRoot` and a title. It does not contain threads: `OrchestrationProject` and `OrchestrationThread` are separate arrays on the read model, and a project can have zero threads. See [workspace-layout.md][2]. diff --git a/docs/user/computer-routing.md b/docs/user/computer-routing.md new file mode 100644 index 000000000000..d0ae01fb229c --- /dev/null +++ b/docs/user/computer-routing.md @@ -0,0 +1,17 @@ +# Sending work to another computer + +Agents can start a new thread on another computer this MT Code client already knows about: + +- `computer_list` returns this machine plus SSH hosts, T3 Connect environments, and other paired + backends, with ids, labels, OS, connection kind, and whether each one is reachable. +- `computer_send` starts a new thread on one of those computers with the given task. + +The current chat is not moved and its transcript is not shared. Include everything the recipient +needs in `message`. Use `computer` set to an environment id, label, SSH `user@host`, or `this`. + +Same-machine sends are handled on that computer's server. Cross-machine sends are brokered by the +desktop, web, or mobile app, which is authenticated to both backends. Keep the app open so it can +complete the hop. Offline SSH hosts are woken the same way as when you pick them in **Run on**. + +`computer_send` returning a thread id means MT Code accepted the new thread on the target computer; +it does not mean that agent finished the work. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 5993fca5b352..caa4d0c6c308 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -122,6 +122,11 @@ npx t3 serve --tailscale-serve --tailscale-serve-port 8443 Once paired, add projects normally: open the Command Palette and choose **Add Project**, then pick the environment the project lives on. Every saved environment is offered, not only the local one. +Agents can also send work to another connected computer without you changing **Run on**. The +`computer_list` and `computer_send` tools start a new thread on a local, SSH, T3 Connect, or +paired environment. The current chat stays put. Keep the desktop or web app open so it can +broker the send. + ### Option 3: Desktop-Managed SSH Launch Use this when you want the desktop app to start or reuse T3 Code on another machine over SSH. diff --git a/docs/user/voice-dictation.md b/docs/user/voice-dictation.md index e34e51a6bdc3..a30121acd1be 100644 --- a/docs/user/voice-dictation.md +++ b/docs/user/voice-dictation.md @@ -1,18 +1,20 @@ # Voice Dictation -Voice dictation records from the message composer and inserts the transcription into your draft. -It is available in the web and desktop clients on browsers that support microphone recording. +Voice dictation follows the Codex composer flow on web, desktop, and the native mobile app: -Enable it in **Settings** → **Beta features** → **Voice dictation**. The composer then shows a -microphone action. Select it to start recording and stop it when you are finished. Recordings stop -automatically after five minutes. +- **X** cancels and discards the recording. +- **Stop** transcribes and appends the text to the end of the current draft. +- **Send** transcribes, appends the text, and uses the normal message-send path. + +Empty and very short recordings are discarded, and recordings stop automatically after five +minutes. A transcript never replaces text that is already in the composer. ## Providers and API Keys -Choose **OpenAI** or **Groq**. T3 Code supplies the provider's transcription endpoint. After an -API key is available, T3 Code loads the models that key can access from the provider and lets you -select the transcription model. Model IDs are not bundled into T3 Code, so newly available models -can appear without an app update. +On web and desktop, open **Settings** → **General** → **Voice dictation**, choose **OpenAI** or +**Groq**, save an API key, and select a transcription model. The microphone appears after the +configuration is complete. T3 Code loads the models available to that key, so new models can +appear without an app update. You can enter a key in the client, or configure it in the environment that runs the connected T3 Code server: @@ -26,3 +28,13 @@ settings. Environment key values are never sent to the client. Recordings are sent through the connected T3 Code server to the selected provider. Recordings larger than 25 MB are rejected. + +## iPhone and Android + +Open **Settings** → **Voice Dictation**, choose **OpenAI** or **Groq**, select a model, and save the +provider's API key. Each provider keeps its own key and model in the device's secure store. OpenAI +defaults to `gpt-4o-transcribe`; Groq defaults to `whisper-large-v3-turbo`. You can also enter a +custom compatible model ID without waiting for an app update. + +Native recordings are sent directly from the mobile app to the selected provider. Clearing the +selected provider's saved key removes the microphone from the composer. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 243a8fb97238..0167334022c4 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -51,6 +51,14 @@ "types": "./src/state/connections.ts", "default": "./src/state/connections.ts" }, + "./state/computers": { + "types": "./src/state/computers.ts", + "default": "./src/state/computers.ts" + }, + "./computers": { + "types": "./src/computers/index.ts", + "default": "./src/computers/index.ts" + }, "./state/entities": { "types": "./src/state/entities.ts", "default": "./src/state/entities.ts" diff --git a/packages/client-runtime/src/computers/catalog.test.ts b/packages/client-runtime/src/computers/catalog.test.ts new file mode 100644 index 000000000000..b815cd908fbf --- /dev/null +++ b/packages/client-runtime/src/computers/catalog.test.ts @@ -0,0 +1,133 @@ +import { EnvironmentId, ProjectId, type OrchestrationProjectShell } from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Option from "effect/Option"; + +import { PrimaryConnectionTarget, SshConnectionTarget } from "../connection/model.ts"; +import { SshConnectionProfile } from "../connection/catalog.ts"; +import { computerPeerFromPresentation } from "./catalog.ts"; +import { matchProject, requireMatchedProject, ComputerTaskDispatchError } from "./matchProject.ts"; + +const now = "2026-01-01T00:00:00.000Z"; + +describe("matchProject", () => { + const projects: ReadonlyArray = [ + { + id: ProjectId.make("project-mac"), + title: "t3code", + workspaceRoot: "/Users/me/dev/t3code", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + { + id: ProjectId.make("project-other"), + title: "notes", + workspaceRoot: "C:/Users/me/notes", + defaultModelSelection: null, + scripts: [], + createdAt: now, + updatedAt: now, + }, + ]; + + it("matches the source project title", () => { + const matched = matchProject({ + projects, + hint: null, + sourceTitle: "t3code", + sourceWorkspaceRoot: "/elsewhere/t3code", + }); + expect("id" in matched && matched.id).toBe("project-mac"); + }); + + it("matches a hint to a workspace basename", () => { + const matched = matchProject({ + projects, + hint: "notes", + sourceTitle: "t3code", + sourceWorkspaceRoot: "/Users/me/dev/t3code", + }); + expect("id" in matched && matched.id).toBe("project-other"); + }); + + it("reports ambiguity when nothing unique matches", () => { + const matched = matchProject({ + projects, + hint: null, + sourceTitle: "unrelated", + sourceWorkspaceRoot: "/tmp/unrelated", + }); + expect("error" in matched && matched.error).toBe("ambiguous"); + expect(() => requireMatchedProject(matched)).toThrow(ComputerTaskDispatchError); + }); +}); + +describe("computerPeerFromPresentation", () => { + it("maps an SSH environment including user@host", () => { + const environmentId = EnvironmentId.make("env-blade"); + const peer = computerPeerFromPresentation({ + entry: { + target: new SshConnectionTarget({ + environmentId, + label: "Blade", + connectionId: "ssh-1", + }), + profile: Option.some( + new SshConnectionProfile({ + connectionId: "ssh-1", + environmentId, + label: "Blade", + target: { + alias: "blade", + hostname: "192.168.50.64", + username: "muhha", + port: 22, + }, + }), + ), + }, + connection: { phase: "connected", error: null, traceId: null }, + serverConfig: { + environment: { + environmentId, + label: "Blade", + platform: { os: "windows", arch: "x64" }, + serverVersion: "0.0.1", + capabilities: {}, + }, + } as never, + }); + expect(peer).toMatchObject({ + environmentId, + label: "Blade", + kind: "ssh", + os: "windows", + connected: true, + sshTarget: "muhha@blade", + }); + }); + + it("maps the primary local environment", () => { + const environmentId = EnvironmentId.make("env-mac"); + const peer = computerPeerFromPresentation({ + entry: { + target: new PrimaryConnectionTarget({ + environmentId, + label: "This device", + httpBaseUrl: "http://127.0.0.1:3773", + wsBaseUrl: "ws://127.0.0.1:3773", + }), + profile: Option.none(), + }, + connection: { phase: "connecting", error: null, traceId: null }, + serverConfig: null, + }); + expect(peer).toMatchObject({ + kind: "local", + os: "unknown", + connected: false, + label: "This device", + }); + }); +}); diff --git a/packages/client-runtime/src/computers/catalog.ts b/packages/client-runtime/src/computers/catalog.ts new file mode 100644 index 000000000000..43eee00582ee --- /dev/null +++ b/packages/client-runtime/src/computers/catalog.ts @@ -0,0 +1,47 @@ +import type { + ComputerKind, + ComputerPeer, + ExecutionEnvironmentPlatformOs, +} from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import { SshConnectionProfile, type ConnectionCatalogEntry } from "../connection/catalog.ts"; +import type { EnvironmentPresentation } from "../connection/presentation.ts"; + +const isSshConnectionProfile = Schema.is(SshConnectionProfile); + +function kindOf(entry: ConnectionCatalogEntry): ComputerKind { + switch (entry.target._tag) { + case "PrimaryConnectionTarget": + return "local"; + case "SshConnectionTarget": + return "ssh"; + case "RelayConnectionTarget": + return "connect"; + case "BearerConnectionTarget": + return "paired"; + } +} + +function sshTargetOf(entry: ConnectionCatalogEntry): string | undefined { + const profile = Option.getOrUndefined(entry.profile); + if (!profile || !isSshConnectionProfile(profile)) return undefined; + const username = profile.target.username?.trim(); + const host = profile.target.alias || profile.target.hostname; + return username && username.length > 0 ? `${username}@${host}` : host; +} + +export function computerPeerFromPresentation(presentation: EnvironmentPresentation): ComputerPeer { + const descriptor = presentation.serverConfig?.environment; + const os: ExecutionEnvironmentPlatformOs = descriptor?.platform.os ?? "unknown"; + const sshTarget = sshTargetOf(presentation.entry); + return { + environmentId: presentation.entry.target.environmentId, + label: descriptor?.label ?? presentation.entry.target.label, + kind: kindOf(presentation.entry), + os, + connected: presentation.connection.phase === "connected", + ...(sshTarget === undefined ? {} : { sshTarget }), + }; +} diff --git a/packages/client-runtime/src/computers/index.ts b/packages/client-runtime/src/computers/index.ts new file mode 100644 index 000000000000..20a9e68debce --- /dev/null +++ b/packages/client-runtime/src/computers/index.ts @@ -0,0 +1,2 @@ +export * from "./catalog.ts"; +export * from "./matchProject.ts"; diff --git a/packages/client-runtime/src/computers/matchProject.ts b/packages/client-runtime/src/computers/matchProject.ts new file mode 100644 index 000000000000..3f7dbbc6fddb --- /dev/null +++ b/packages/client-runtime/src/computers/matchProject.ts @@ -0,0 +1,108 @@ +import type { OrchestrationProjectShell } from "@t3tools/contracts"; + +export type ComputerTaskDispatchErrorCode = + | "dispatch_failed" + | "project_not_found" + | "project_ambiguous" + | "computer_offline"; + +export class ComputerTaskDispatchError extends Error { + readonly code: ComputerTaskDispatchErrorCode; + + constructor(code: ComputerTaskDispatchErrorCode, detail: string) { + super(detail); + this.name = "ComputerTaskDispatchError"; + this.code = code; + } +} + +function basename(path: string): string { + const trimmed = path.replaceAll("\\", "/").replace(/\/+$/, ""); + const parts = trimmed.split("/"); + return parts[parts.length - 1] ?? trimmed; +} + +function normalize(value: string): string { + return value.trim().toLowerCase(); +} + +export function matchProject(input: { + readonly projects: ReadonlyArray; + readonly hint: string | null; + readonly sourceTitle: string; + readonly sourceWorkspaceRoot: string; +}): + | OrchestrationProjectShell + | { readonly error: "not_found" | "ambiguous"; readonly detail: string } { + const projects = input.projects; + const describe = (project: OrchestrationProjectShell) => + `${project.title} (${project.workspaceRoot})`; + + const pick = ( + candidates: ReadonlyArray, + emptyDetail: string, + manyDetail: string, + ) => { + if (candidates.length === 1) return candidates[0]!; + if (candidates.length === 0) return { error: "not_found" as const, detail: emptyDetail }; + return { + error: "ambiguous" as const, + detail: `${manyDetail}: ${candidates.map(describe).join("; ")}`, + }; + }; + + if (input.hint && input.hint.trim().length > 0) { + const needle = normalize(input.hint); + const exactTitle = projects.filter((project) => normalize(project.title) === needle); + if (exactTitle.length === 1) return exactTitle[0]!; + const exactPath = projects.filter( + (project) => + normalize(project.workspaceRoot) === needle || + normalize(basename(project.workspaceRoot)) === needle, + ); + if (exactPath.length === 1) return exactPath[0]!; + const includes = projects.filter( + (project) => + normalize(project.title).includes(needle) || + normalize(project.workspaceRoot).includes(needle), + ); + return pick( + includes, + `No project on that computer matches '${input.hint}'.`, + `Multiple projects match '${input.hint}'`, + ); + } + + const byTitle = projects.filter( + (project) => normalize(project.title) === normalize(input.sourceTitle), + ); + if (byTitle.length === 1) return byTitle[0]!; + const byBasename = projects.filter( + (project) => + normalize(basename(project.workspaceRoot)) === normalize(basename(input.sourceWorkspaceRoot)), + ); + if (byBasename.length === 1) return byBasename[0]!; + if (projects.length === 1) return projects[0]!; + if (projects.length === 0) { + return { + error: "not_found", + detail: "That computer has no T3 projects yet. Add a project there first.", + }; + } + return { + error: "ambiguous", + detail: `Could not match a project on that computer. Pass project: ${projects.map(describe).join("; ")}`, + }; +} + +export function requireMatchedProject( + matched: ReturnType, +): OrchestrationProjectShell { + if ("error" in matched) { + throw new ComputerTaskDispatchError( + matched.error === "not_found" ? "project_not_found" : "project_ambiguous", + matched.detail, + ); + } + return matched; +} diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index bdfb2ff77383..02201b2f427c 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -52,6 +52,7 @@ export type EnvironmentSubscriptionRpcTag = | typeof WS_METHODS.subscribeDiscoveredLocalServers | typeof WS_METHODS.subscribeResourceTelemetry | typeof WS_METHODS.previewAutomationConnect + | typeof WS_METHODS.computersConnect | typeof WS_METHODS.subscribeVcsStatus | typeof WS_METHODS.terminalAttach; diff --git a/packages/client-runtime/src/state/computers.ts b/packages/client-runtime/src/state/computers.ts new file mode 100644 index 000000000000..21672a0dc40c --- /dev/null +++ b/packages/client-runtime/src/state/computers.ts @@ -0,0 +1,41 @@ +import { WS_METHODS } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { + createAtomCommandScheduler, + createEnvironmentRpcCommand, + createEnvironmentRpcSubscriptionAtomFamily, +} from "./runtime.ts"; + +export function createComputerEnvironmentAtoms( + runtime: Atom.AtomRuntime, +) { + const scheduler = createAtomCommandScheduler(); + return { + requests: createEnvironmentRpcSubscriptionAtomFamily(runtime, { + label: "environment-data:computers:requests", + tag: WS_METHODS.computersConnect, + idleTtlMs: 0, + }), + sync: createEnvironmentRpcCommand(runtime, { + label: "environment-data:computers:sync", + tag: WS_METHODS.computersSync, + scheduler, + concurrency: { + mode: "latest", + key: ({ environmentId, input }) => JSON.stringify([environmentId, input.clientId]), + }, + }), + respond: createEnvironmentRpcCommand(runtime, { + label: "environment-data:computers:respond", + tag: WS_METHODS.computersRespond, + scheduler, + concurrency: { + mode: "singleFlight", + key: ({ environmentId, input }) => + JSON.stringify([environmentId, input.connectionId, input.requestId]), + }, + }), + }; +} diff --git a/packages/contracts/src/computers.ts b/packages/contracts/src/computers.ts new file mode 100644 index 000000000000..585efbfcb042 --- /dev/null +++ b/packages/contracts/src/computers.ts @@ -0,0 +1,131 @@ +import { EnvironmentId, ProjectId, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { ExecutionEnvironmentPlatformOs } from "./environment.ts"; +import { ModelSelection, ProviderInteractionMode, RuntimeMode } from "./orchestration.ts"; +import * as Schema from "effect/Schema"; + +export const COMPUTER_SEND_MESSAGE_MAX_CHARS = 8_000; +export const COMPUTER_SEND_TIMEOUT_MS = 45_000; + +export const ComputerKind = Schema.Literals(["local", "ssh", "connect", "paired"]); +export type ComputerKind = typeof ComputerKind.Type; + +export const ComputerPeer = Schema.Struct({ + environmentId: EnvironmentId, + label: TrimmedNonEmptyString, + kind: ComputerKind, + os: ExecutionEnvironmentPlatformOs, + connected: Schema.Boolean, + sshTarget: Schema.optional(TrimmedNonEmptyString), +}); +export type ComputerPeer = typeof ComputerPeer.Type; + +export const ComputerListEntry = Schema.Struct({ + environmentId: EnvironmentId, + label: TrimmedNonEmptyString, + kind: ComputerKind, + os: ExecutionEnvironmentPlatformOs, + connected: Schema.Boolean, + sshTarget: Schema.optional(TrimmedNonEmptyString), + thisMachine: Schema.Boolean, +}); +export type ComputerListEntry = typeof ComputerListEntry.Type; + +export const ComputerTaskClientId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type ComputerTaskClientId = typeof ComputerTaskClientId.Type; + +export const ComputerTaskConnectionId = TrimmedNonEmptyString.check(Schema.isMaxLength(128)); +export type ComputerTaskConnectionId = typeof ComputerTaskConnectionId.Type; + +export const ComputerTaskHost = Schema.Struct({ + clientId: ComputerTaskClientId, + computers: Schema.Array(ComputerPeer), +}); +export type ComputerTaskHost = typeof ComputerTaskHost.Type; + +export const ComputerTaskSource = Schema.Struct({ + environmentId: EnvironmentId, + threadId: ThreadId, + label: TrimmedNonEmptyString, + projectTitle: TrimmedNonEmptyString, + projectWorkspaceRoot: TrimmedNonEmptyString, +}); +export type ComputerTaskSource = typeof ComputerTaskSource.Type; + +export const ComputerTaskSendRequest = Schema.Struct({ + requestId: TrimmedNonEmptyString, + computer: ComputerPeer, + message: TrimmedNonEmptyString, + title: TrimmedNonEmptyString, + source: ComputerTaskSource, + projectHint: Schema.NullOr(TrimmedNonEmptyString), + modelSelection: ModelSelection, + runtimeMode: RuntimeMode, + interactionMode: ProviderInteractionMode, +}); +export type ComputerTaskSendRequest = typeof ComputerTaskSendRequest.Type; + +export const ComputerTaskSendResult = Schema.Struct({ + environmentId: EnvironmentId, + threadId: ThreadId, + projectId: ProjectId, +}); +export type ComputerTaskSendResult = typeof ComputerTaskSendResult.Type; + +export const ComputerTaskStreamEvent = Schema.Union([ + Schema.Struct({ + type: Schema.Literal("connected"), + connectionId: ComputerTaskConnectionId, + }), + Schema.Struct({ + type: Schema.Literal("request"), + connectionId: ComputerTaskConnectionId, + request: ComputerTaskSendRequest, + }), +]); +export type ComputerTaskStreamEvent = typeof ComputerTaskStreamEvent.Type; + +export const ComputerTaskErrorCode = Schema.Literals([ + "source_unavailable", + "computer_not_found", + "computer_offline", + "no_client", + "project_not_found", + "project_ambiguous", + "query_failed", + "dispatch_failed", +]); +export type ComputerTaskErrorCode = typeof ComputerTaskErrorCode.Type; + +export class ComputerTaskError extends Schema.TaggedErrorClass()( + "ComputerTaskError", + { + code: ComputerTaskErrorCode, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + return this.detail; + } +} + +export const ComputerTaskResponse = Schema.Struct({ + clientId: ComputerTaskClientId, + connectionId: ComputerTaskConnectionId, + requestId: TrimmedNonEmptyString, + ok: Schema.Boolean, + result: Schema.optional(ComputerTaskSendResult), + error: Schema.optional( + Schema.Struct({ + code: ComputerTaskErrorCode, + detail: Schema.String, + }), + ), +}); +export type ComputerTaskResponse = typeof ComputerTaskResponse.Type; + +export const ComputerListResult = Schema.Struct({ + thisEnvironmentId: EnvironmentId, + computers: Schema.Array(ComputerListEntry), +}); +export type ComputerListResult = typeof ComputerListResult.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index f38a4a6a9c8a..b02d2f860337 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -16,6 +16,7 @@ export * from "./keybindings.ts"; export * from "./server.ts"; export * from "./settings.ts"; export * from "./computerHistory.ts"; +export * from "./computers.ts"; export * from "./git.ts"; export * from "./vcs.ts"; export * from "./sourceControl.ts"; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index bf8bb80cfa90..05855c3850d9 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -170,6 +170,12 @@ import { PreviewAutomationResponse, PreviewAutomationStreamEvent, } from "./previewAutomation.ts"; +import { + ComputerTaskError, + ComputerTaskHost, + ComputerTaskResponse, + ComputerTaskStreamEvent, +} from "./computers.ts"; import { ServerConfigStreamEvent, ServerConfig, @@ -270,6 +276,9 @@ export const WS_METHODS = { previewAutomationConnect: "previewAutomation.connect", previewAutomationRespond: "previewAutomation.respond", previewAutomationFocusHost: "previewAutomation.focusHost", + computersConnect: "computers.connect", + computersSync: "computers.sync", + computersRespond: "computers.respond", // Server meta serverProbe: "server.probe", @@ -959,6 +968,23 @@ export const WsPreviewAutomationFocusHostRpc = Rpc.make(WS_METHODS.previewAutoma error: EnvironmentAuthorizationError, }); +export const WsComputersConnectRpc = Rpc.make(WS_METHODS.computersConnect, { + payload: ComputerTaskHost, + success: ComputerTaskStreamEvent, + error: Schema.Union([ComputerTaskError, EnvironmentAuthorizationError]), + stream: true, +}); + +export const WsComputersSyncRpc = Rpc.make(WS_METHODS.computersSync, { + payload: ComputerTaskHost, + error: Schema.Union([ComputerTaskError, EnvironmentAuthorizationError]), +}); + +export const WsComputersRespondRpc = Rpc.make(WS_METHODS.computersRespond, { + payload: ComputerTaskResponse, + error: Schema.Union([ComputerTaskError, EnvironmentAuthorizationError]), +}); + export const WsSubscribePreviewEventsRpc = Rpc.make(WS_METHODS.subscribePreviewEvents, { payload: Schema.Struct({}), success: PreviewEvent, @@ -1186,6 +1212,9 @@ export const WsRpcGroup = RpcGroup.make( WsPreviewAutomationConnectRpc, WsPreviewAutomationRespondRpc, WsPreviewAutomationFocusHostRpc, + WsComputersConnectRpc, + WsComputersSyncRpc, + WsComputersRespondRpc, WsSubscribePreviewEventsRpc, WsSubscribeDiscoveredLocalServersRpc, WsSubscribeServerConfigRpc, diff --git a/packages/shared/package.json b/packages/shared/package.json index 4db7d9ccbbe2..a6d49349559f 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -183,6 +183,10 @@ "types": "./src/composerInlineTokens.ts", "import": "./src/composerInlineTokens.ts" }, + "./voiceTranscription": { + "types": "./src/voiceTranscription.ts", + "import": "./src/voiceTranscription.ts" + }, "./terminalLabels": { "types": "./src/terminalLabels.ts", "import": "./src/terminalLabels.ts" diff --git a/packages/shared/src/voiceTranscription.test.ts b/packages/shared/src/voiceTranscription.test.ts new file mode 100644 index 000000000000..5ebe1daf0f14 --- /dev/null +++ b/packages/shared/src/voiceTranscription.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { appendVoiceTranscript, resolveVoiceTranscriptionAction } from "./voiceTranscription.js"; + +describe("appendVoiceTranscript", () => { + it("appends trimmed speech to an empty draft", () => { + expect(appendVoiceTranscript("", " hello ")).toBe("hello"); + }); + + it("adds one boundary space after existing text", () => { + expect(appendVoiceTranscript("existing", "speech")).toBe("existing speech"); + }); + + it("preserves an existing whitespace boundary", () => { + expect(appendVoiceTranscript("existing\n", " speech ")).toBe("existing\nspeech"); + }); + + it("ignores an empty transcript", () => { + expect(appendVoiceTranscript("existing", " ")).toBe("existing"); + }); +}); + +describe("resolveVoiceTranscriptionAction", () => { + it("upgrades insert to send", () => { + expect(resolveVoiceTranscriptionAction("insert", "send")).toBe("send"); + }); + + it("does not downgrade send to insert", () => { + expect(resolveVoiceTranscriptionAction("send", "insert")).toBe("send"); + }); + + it("lets cancellation win", () => { + expect(resolveVoiceTranscriptionAction("send", "abort")).toBe("abort"); + expect(resolveVoiceTranscriptionAction("abort", "send")).toBe("abort"); + }); +}); diff --git a/packages/shared/src/voiceTranscription.ts b/packages/shared/src/voiceTranscription.ts new file mode 100644 index 000000000000..64990c3b0abe --- /dev/null +++ b/packages/shared/src/voiceTranscription.ts @@ -0,0 +1,26 @@ +/** + * Mirrors the Codex composer boundary rule: trim the transcript, append it to + * the end, and add exactly one space only when the existing draft needs one. + */ +export function appendVoiceTranscript(existing: string, transcript: string): string { + const normalized = transcript.trim(); + if (normalized.length === 0) return existing; + if (existing.length === 0 || /\s$/.test(existing)) return `${existing}${normalized}`; + return `${existing} ${normalized}`; +} + +export type VoiceTranscriptionAction = "insert" | "send" | "abort"; + +/** + * Keeps a single terminal outcome for a recording. Cancellation always wins; + * pressing Send can upgrade an already-requested insert while transcription is + * being finalized. + */ +export function resolveVoiceTranscriptionAction( + current: VoiceTranscriptionAction | null, + next: VoiceTranscriptionAction, +): VoiceTranscriptionAction { + if (current === "abort" || next === "abort") return "abort"; + if (current === "send" || next === "send") return "send"; + return "insert"; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad9d72383895..3970f1ede3e2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -289,6 +289,9 @@ importers: expo-asset: specifier: ~56.0.17 version: 56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + expo-audio: + specifier: ~56.0.13 + version: 56.0.13(expo-asset@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) expo-auth-session: specifier: ~56.0.14 version: 56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -6629,6 +6632,14 @@ packages: react: '*' react-native: '*' + expo-audio@56.0.13: + resolution: {integrity: sha512-pfBmT/8OYbhrb37ECk+fC1EstLBFxMTioF4uZrAEeOj3uCgjproj1qYmkpsQLJIoEOMPSIw1iSfBEMl10v1DDA==} + peerDependencies: + expo: '*' + expo-asset: '*' + react: '*' + react-native: '*' + expo-auth-session@56.0.14: resolution: {integrity: sha512-b6URDBKXVWBjHwypnbCPW6A3PrwYyFqzLXtTrrpTGpmDlsxk7xuz6wIA77sBNziz3hMxt11Nu70iY0fJZYT4jA==} peerDependencies: @@ -16969,6 +16980,13 @@ snapshots: - typescript optional: true + expo-audio@56.0.13(expo-asset@56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): + dependencies: + expo: 56.0.12(8895228379997a2a064f9644cda56ed0) + expo-asset: 56.0.17(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + expo-auth-session@56.0.14(expo@56.0.12)(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): dependencies: expo-application: 56.0.3(expo@56.0.12)