diff --git a/frontend/components/VoiceAgent.tsx b/frontend/components/VoiceAgent.tsx index e611d1f..bad9126 100644 --- a/frontend/components/VoiceAgent.tsx +++ b/frontend/components/VoiceAgent.tsx @@ -7,6 +7,7 @@ import { ClaudeBackend, RealtimeEvent, RealtimeOptions, VoiceProvider, VoiceSess import { INSTRUCTIONS } from "@/lib/instructions"; import { authHeaders, withAuthParam } from "@/lib/auth"; import { scopedClearPending } from "@/lib/promptState"; +import { orbTarget, envelopeStep } from "@/lib/orb"; import LiveTerminal from "./LiveTerminal"; import { Icon } from "./ui/Icon"; @@ -499,10 +500,10 @@ export default function VoiceAgent() { new Map(), ); const smoothedRef = useRef(0); - // Mirror `connected` into a ref so the rAF orb loop (a stable closure) can gate - // volume scaling without re-subscribing — the orb stays still until fully - // connected, even though the mic analyser attaches during the handshake. - const connectedRef = useRef(false); + // The rAF orb loop gates volume scaling on this ref: the orb stays still until + // the transport fires `ready` (see voice.ts), even though the mic analyser + // attaches earlier, during the connection handshake. + const readyRef = useRef(false); const pollTimers = useRef>>(new Map()); const txPollRef = useRef | null>(null); // Cost-log connection identity & snapshot timer. connectionId persists for one @@ -983,35 +984,19 @@ export default function VoiceAgent() { return () => window.removeEventListener("resize", updateConvFades); }, [updateConvFades]); - // Keep the orb-loop's connection gate current. - useEffect(() => { - connectedRef.current = connected; - }, [connected]); - // Drive the orb's size from live audio volume. Both the user's mic and the // assistant's speech feed analysers on ONE shared AudioContext; each frame we // take the LOUDER of the two as the instantaneous target, then envelope-smooth // it (fast attack, slow release) so the orb rises lively and falls gently // instead of twitching frame-to-frame. const orbLoop = () => { - let target = 0; - // Only react to audio once fully connected; while connecting the analyser is - // already live (mic acquired) but the orb should stay at rest (target 0). - if (connectedRef.current) { - for (const { analyser, buf } of analysersRef.current.values()) { - analyser.getByteTimeDomainData(buf); - let sum = 0; - for (let i = 0; i < buf.length; i++) { - const v = (buf[i] - 128) / 128; - sum += v * v; - } - const rms = Math.min(1, Math.sqrt(sum / buf.length) * 3.2); - if (rms > target) target = rms; // loudest source wins - } + const bufs: Uint8Array[] = []; + for (const { analyser, buf } of analysersRef.current.values()) { + analyser.getByteTimeDomainData(buf); + bufs.push(buf); } - // Envelope: snap up quickly, ease down slowly. - const k = target > smoothedRef.current ? 0.35 : 0.08; - smoothedRef.current += (target - smoothedRef.current) * k; + const target = orbTarget(readyRef.current, bufs); + smoothedRef.current = envelopeStep(smoothedRef.current, target); const amp = smoothedRef.current.toFixed(3); orbRef.current?.style.setProperty("--amp", amp); glowRef.current?.style.setProperty("--amp", amp); @@ -1060,7 +1045,14 @@ export default function VoiceAgent() { const onEvent = (e: RealtimeEvent) => { if (e.type === "status") setStatus(e.status); - else if (e.type === "state") setVstate(e.state); + else if (e.type === "ready") readyRef.current = true; // open the orb gate + else if (e.type === "state") { + // Dropping back to "connecting" (a Gemini reconnect) means the session is + // no longer ready — close the orb gate until the next `ready` re-opens it, + // so the orb rests across the outage instead of scaling from mic input. + if (e.state === "connecting") readyRef.current = false; + setVstate(e.state); + } else if (e.type === "usage") setVoiceUsage(e.usage); else if (e.type === "error") { setStatus(`Error: ${e.message}`); @@ -1172,6 +1164,7 @@ export default function VoiceAgent() { const connect = async () => { setVstate("connecting"); + readyRef.current = false; // Fetch the session list fresh rather than trusting the 2s poll — on a cold // page load the polled state may still be empty, and baking a wrong "no // sessions" snapshot invites the model to start duplicates. @@ -1252,6 +1245,7 @@ export default function VoiceAgent() { sessionRef.current?.stop(); sessionRef.current = null; + readyRef.current = false; stopAnalyser(); stopPolling(); if (txPollRef.current) { diff --git a/frontend/lib/gemini.ts b/frontend/lib/gemini.ts index f9e5f84..a057559 100644 --- a/frontend/lib/gemini.ts +++ b/frontend/lib/gemini.ts @@ -441,6 +441,9 @@ export class GeminiSession implements VoiceSession { this.startKeepalive(); this.onSetupComplete?.(); // unblocks a reconnect attempt waiting on setup emit({ type: "status", status: "Connected — start talking." }); + // setupComplete is Gemini's genuine ready point — start() returns earlier, + // before the handshake finishes. + emit({ type: "ready" }); emit({ type: "state", state: "listening" }); return; } diff --git a/frontend/lib/orb.test.ts b/frontend/lib/orb.test.ts new file mode 100644 index 0000000..66123f6 --- /dev/null +++ b/frontend/lib/orb.test.ts @@ -0,0 +1,53 @@ +// Orb gate-timing tests. Run with Node's built-in runner (no deps): +// node --test lib/orb.test.ts # from frontend/ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { rmsAmp, orbTarget, envelopeStep } from "./orb.ts"; + +// A buffer pegged at 128 is pure silence (the byte time-domain midpoint). +const SILENCE = new Uint8Array(256).fill(128); +// A loud buffer alternating between the extremes — well above the gate. +const LOUD = Uint8Array.from({ length: 256 }, (_, i) => (i % 2 ? 255 : 0)); + +test("silence reads as zero amplitude", () => { + assert.equal(rmsAmp(SILENCE), 0); +}); + +test("loud input produces a non-zero amplitude (clamped to 1)", () => { + const amp = rmsAmp(LOUD); + assert.ok(amp > 0); + assert.ok(amp <= 1); +}); + +test("CORE GATE: a closed gate pins the target to 0 even with loud input", () => { + // This is the bug fix: before `ready`, the mic analyser is already live but + // the orb must not scale. The gate (readyRef) keeps the target at 0. + assert.equal(orbTarget(false, [LOUD, LOUD]), 0); +}); + +test("an open gate scales from the loudest analyser", () => { + // Loudest source wins: silence + loud => loud's amplitude. + assert.equal(orbTarget(true, [SILENCE, LOUD]), rmsAmp(LOUD)); +}); + +test("an open gate over only silence is still 0", () => { + assert.equal(orbTarget(true, [SILENCE]), 0); +}); + +test("the envelope stays at rest while the gate keeps the target at 0", () => { + // Drive several frames with a closed gate from a non-zero starting point: the + // envelope must decay toward 0 and never be pushed up by mic input. + let smoothed = 0.5; + for (let i = 0; i < 100; i++) { + smoothed = envelopeStep(smoothed, orbTarget(false, [LOUD])); + } + assert.ok(smoothed < 1e-3, `expected near-0, got ${smoothed}`); +}); + +test("the envelope rises once the gate opens", () => { + let smoothed = 0; + const target = orbTarget(true, [LOUD]); + for (let i = 0; i < 20; i++) smoothed = envelopeStep(smoothed, target); + assert.ok(smoothed > 0.1, `expected the orb to rise, got ${smoothed}`); +}); diff --git a/frontend/lib/orb.ts b/frontend/lib/orb.ts new file mode 100644 index 0000000..c366eef --- /dev/null +++ b/frontend/lib/orb.ts @@ -0,0 +1,38 @@ +// Pure helpers for the voice orb's volume animation, extracted from the React +// component so the gate timing is unit-testable (see orb.test.ts). +// +// The orb scales with live audio: each rAF frame we read the time-domain samples +// from every active analyser (the user's mic and the assistant's speech), take +// the loudest as the instantaneous target, then envelope-smooth it. The gate +// keeps the target pinned at 0 until the session is genuinely ready — the mic +// analyser attaches during the connection handshake, so without the gate the orb +// would scale from mic input before the agent can actually hear/respond. + +// RMS amplitude of one analyser's byte time-domain buffer, scaled into 0..1. +export function rmsAmp(buf: Uint8Array | number[]): number { + let sum = 0; + for (let i = 0; i < buf.length; i++) { + const v = (buf[i] - 128) / 128; + sum += v * v; + } + return Math.min(1, Math.sqrt(sum / buf.length) * 3.2); +} + +// Instantaneous orb target for one frame. Returns 0 when the gate is closed +// (still connecting), otherwise the loudest analyser's RMS. +export function orbTarget(gateOpen: boolean, bufs: Array): number { + if (!gateOpen) return 0; + let target = 0; + for (const buf of bufs) { + const rms = rmsAmp(buf); + if (rms > target) target = rms; // loudest source wins + } + return target; +} + +// Envelope step: snap up quickly (fast attack), ease down slowly (slow release) +// so the orb rises lively and falls gently instead of twitching frame-to-frame. +export function envelopeStep(smoothed: number, target: number): number { + const k = target > smoothed ? 0.35 : 0.08; + return smoothed + (target - smoothed) * k; +} diff --git a/frontend/lib/realtime.ts b/frontend/lib/realtime.ts index 376b0a3..75954a0 100644 --- a/frontend/lib/realtime.ts +++ b/frontend/lib/realtime.ts @@ -139,7 +139,15 @@ export class RealtimeSession implements VoiceSession { const dc = pc.createDataChannel("oai-events"); this.dc = dc; - dc.addEventListener("open", () => this.configureSession()); + // The data channel opens after start() resolves and ICE connects. Fire the + // ready/listening signals here, once configureSession has run — not at the + // end of start(), which runs before the channel is open. + dc.addEventListener("open", () => { + this.configureSession(); + emit({ type: "status", status: "Connected — start talking." }); + emit({ type: "ready" }); + emit({ type: "state", state: "listening" }); + }); dc.addEventListener("message", (ev) => this.handleEvent(ev.data)); const offer = await pc.createOffer(); @@ -156,9 +164,7 @@ export class RealtimeSession implements VoiceSession { }); if (!sdpResp.ok) throw new Error(`SDP exchange failed: ${sdpResp.status} ${await sdpResp.text()}`); await pc.setRemoteDescription({ type: "answer", sdp: await sdpResp.text() }); - - emit({ type: "status", status: "Connected — start talking." }); - emit({ type: "state", state: "listening" }); + // ready/listening fire from the data channel `open` handler above, not here. } stop(): void { diff --git a/frontend/lib/voice.ts b/frontend/lib/voice.ts index bbb5f96..3d2abb6 100644 --- a/frontend/lib/voice.ts +++ b/frontend/lib/voice.ts @@ -34,6 +34,10 @@ export type VoiceUsage = { export type RealtimeEvent = | { type: "status"; status: string } | { type: "state"; state: VoiceState } + // Fired at the genuine ready point — the post-start() handshake completing + // (Gemini: setupComplete; OpenAI/Azure: data channel open + configured). The + // orb's mic-volume animation gates on this, since start() resolving is not ready. + | { type: "ready" } | { type: "transcript"; role: "user" | "assistant"; text: string; final: boolean } | { type: "tool_call"; name: string; arguments?: unknown; result?: unknown; ok?: boolean } | { type: "usage"; usage: VoiceUsage }