fix(ui): gate voice orb on a true ready signal, not start() returning (#38)

frontend/components/VoiceAgent.tsx

@@ -7,6 +7,7 @@ import { ClaudeBackend, RealtimeEvent, RealtimeOptions, VoiceProvider, VoiceSess
 import { INSTRUCTIONS } from "@/lib/instructions";
 import { authHeaders, withAuthParam } from "@/lib/auth";
 import { scopedClearPending } from "@/lib/promptState";
+import { orbTarget, envelopeStep } from "@/lib/orb";
 import LiveTerminal from "./LiveTerminal";
 import { Icon } from "./ui/Icon";
 
@@ -499,10 +500,10 @@ export default function VoiceAgent() {
     new Map(),
   );
   const smoothedRef = useRef(0);
-  // Mirror `connected` into a ref so the rAF orb loop (a stable closure) can gate
-  // volume scaling without re-subscribing — the orb stays still until fully
-  // connected, even though the mic analyser attaches during the handshake.
-  const connectedRef = useRef(false);
+  // The rAF orb loop gates volume scaling on this ref: the orb stays still until
+  // the transport fires `ready` (see voice.ts), even though the mic analyser
+  // attaches earlier, during the connection handshake.
+  const readyRef = useRef(false);
   const pollTimers = useRef<Map<string, ReturnType<typeof setInterval>>>(new Map());
   const txPollRef = useRef<ReturnType<typeof setInterval> | null>(null);
   // Cost-log connection identity & snapshot timer. connectionId persists for one
@@ -983,35 +984,19 @@ export default function VoiceAgent() {
     return () => window.removeEventListener("resize", updateConvFades);
   }, [updateConvFades]);
 
-  // Keep the orb-loop's connection gate current.
-  useEffect(() => {
-    connectedRef.current = connected;
-  }, [connected]);
-
   // Drive the orb's size from live audio volume. Both the user's mic and the
   // assistant's speech feed analysers on ONE shared AudioContext; each frame we
   // take the LOUDER of the two as the instantaneous target, then envelope-smooth
   // it (fast attack, slow release) so the orb rises lively and falls gently
   // instead of twitching frame-to-frame.
   const orbLoop = () => {
-    let target = 0;
-    // Only react to audio once fully connected; while connecting the analyser is
-    // already live (mic acquired) but the orb should stay at rest (target 0).
-    if (connectedRef.current) {
-      for (const { analyser, buf } of analysersRef.current.values()) {
-        analyser.getByteTimeDomainData(buf);
-        let sum = 0;
-        for (let i = 0; i < buf.length; i++) {
-          const v = (buf[i] - 128) / 128;
-          sum += v * v;
-        }
-        const rms = Math.min(1, Math.sqrt(sum / buf.length) * 3.2);
-        if (rms > target) target = rms; // loudest source wins
-      }
+    const bufs: Uint8Array[] = [];
+    for (const { analyser, buf } of analysersRef.current.values()) {
+      analyser.getByteTimeDomainData(buf);
+      bufs.push(buf);
     }
-    // Envelope: snap up quickly, ease down slowly.
-    const k = target > smoothedRef.current ? 0.35 : 0.08;
-    smoothedRef.current += (target - smoothedRef.current) * k;
+    const target = orbTarget(readyRef.current, bufs);
+    smoothedRef.current = envelopeStep(smoothedRef.current, target);
     const amp = smoothedRef.current.toFixed(3);
     orbRef.current?.style.setProperty("--amp", amp);
     glowRef.current?.style.setProperty("--amp", amp);
@@ -1060,7 +1045,14 @@ export default function VoiceAgent() {
 
   const onEvent = (e: RealtimeEvent) => {
     if (e.type === "status") setStatus(e.status);
-    else if (e.type === "state") setVstate(e.state);
+    else if (e.type === "ready") readyRef.current = true; // open the orb gate
+    else if (e.type === "state") {
+      // Dropping back to "connecting" (a Gemini reconnect) means the session is
+      // no longer ready — close the orb gate until the next `ready` re-opens it,
+      // so the orb rests across the outage instead of scaling from mic input.
+      if (e.state === "connecting") readyRef.current = false;
+      setVstate(e.state);
+    }
     else if (e.type === "usage") setVoiceUsage(e.usage);
     else if (e.type === "error") {
       setStatus(`Error: ${e.message}`);
@@ -1172,6 +1164,7 @@ export default function VoiceAgent() {
 
   const connect = async () => {
     setVstate("connecting");
+    readyRef.current = false;
     // Fetch the session list fresh rather than trusting the 2s poll — on a cold
     // page load the polled state may still be empty, and baking a wrong "no
     // sessions" snapshot invites the model to start duplicates.
@@ -1252,6 +1245,7 @@ export default function VoiceAgent() {
 
     sessionRef.current?.stop();
     sessionRef.current = null;
+    readyRef.current = false;
     stopAnalyser();
     stopPolling();
     if (txPollRef.current) {

frontend/lib/gemini.ts

@@ -441,6 +441,9 @@ export class GeminiSession implements VoiceSession {
       this.startKeepalive();
       this.onSetupComplete?.();  // unblocks a reconnect attempt waiting on setup
       emit({ type: "status", status: "Connected — start talking." });
+      // setupComplete is Gemini's genuine ready point — start() returns earlier,
+      // before the handshake finishes.
+      emit({ type: "ready" });
       emit({ type: "state", state: "listening" });
       return;
     }

frontend/lib/orb.test.ts

@@ -0,0 +1,53 @@
+// Orb gate-timing tests. Run with Node's built-in runner (no deps):
+//     node --test lib/orb.test.ts     # from frontend/
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { rmsAmp, orbTarget, envelopeStep } from "./orb.ts";
+
+// A buffer pegged at 128 is pure silence (the byte time-domain midpoint).
+const SILENCE = new Uint8Array(256).fill(128);
+// A loud buffer alternating between the extremes — well above the gate.
+const LOUD = Uint8Array.from({ length: 256 }, (_, i) => (i % 2 ? 255 : 0));
+
+test("silence reads as zero amplitude", () => {
+  assert.equal(rmsAmp(SILENCE), 0);
+});
+
+test("loud input produces a non-zero amplitude (clamped to 1)", () => {
+  const amp = rmsAmp(LOUD);
+  assert.ok(amp > 0);
+  assert.ok(amp <= 1);
+});
+
+test("CORE GATE: a closed gate pins the target to 0 even with loud input", () => {
+  // This is the bug fix: before `ready`, the mic analyser is already live but
+  // the orb must not scale. The gate (readyRef) keeps the target at 0.
+  assert.equal(orbTarget(false, [LOUD, LOUD]), 0);
+});
+
+test("an open gate scales from the loudest analyser", () => {
+  // Loudest source wins: silence + loud => loud's amplitude.
+  assert.equal(orbTarget(true, [SILENCE, LOUD]), rmsAmp(LOUD));
+});
+
+test("an open gate over only silence is still 0", () => {
+  assert.equal(orbTarget(true, [SILENCE]), 0);
+});
+
+test("the envelope stays at rest while the gate keeps the target at 0", () => {
+  // Drive several frames with a closed gate from a non-zero starting point: the
+  // envelope must decay toward 0 and never be pushed up by mic input.
+  let smoothed = 0.5;
+  for (let i = 0; i < 100; i++) {
+    smoothed = envelopeStep(smoothed, orbTarget(false, [LOUD]));
+  }
+  assert.ok(smoothed < 1e-3, `expected near-0, got ${smoothed}`);
+});
+
+test("the envelope rises once the gate opens", () => {
+  let smoothed = 0;
+  const target = orbTarget(true, [LOUD]);
+  for (let i = 0; i < 20; i++) smoothed = envelopeStep(smoothed, target);
+  assert.ok(smoothed > 0.1, `expected the orb to rise, got ${smoothed}`);
+});

frontend/lib/orb.ts

@@ -0,0 +1,38 @@
+// Pure helpers for the voice orb's volume animation, extracted from the React
+// component so the gate timing is unit-testable (see orb.test.ts).
+//
+// The orb scales with live audio: each rAF frame we read the time-domain samples
+// from every active analyser (the user's mic and the assistant's speech), take
+// the loudest as the instantaneous target, then envelope-smooth it. The gate
+// keeps the target pinned at 0 until the session is genuinely ready — the mic
+// analyser attaches during the connection handshake, so without the gate the orb
+// would scale from mic input before the agent can actually hear/respond.
+
+// RMS amplitude of one analyser's byte time-domain buffer, scaled into 0..1.
+export function rmsAmp(buf: Uint8Array | number[]): number {
+  let sum = 0;
+  for (let i = 0; i < buf.length; i++) {
+    const v = (buf[i] - 128) / 128;
+    sum += v * v;
+  }
+  return Math.min(1, Math.sqrt(sum / buf.length) * 3.2);
+}
+
+// Instantaneous orb target for one frame. Returns 0 when the gate is closed
+// (still connecting), otherwise the loudest analyser's RMS.
+export function orbTarget(gateOpen: boolean, bufs: Array<Uint8Array | number[]>): number {
+  if (!gateOpen) return 0;
+  let target = 0;
+  for (const buf of bufs) {
+    const rms = rmsAmp(buf);
+    if (rms > target) target = rms; // loudest source wins
+  }
+  return target;
+}
+
+// Envelope step: snap up quickly (fast attack), ease down slowly (slow release)
+// so the orb rises lively and falls gently instead of twitching frame-to-frame.
+export function envelopeStep(smoothed: number, target: number): number {
+  const k = target > smoothed ? 0.35 : 0.08;
+  return smoothed + (target - smoothed) * k;
+}

frontend/lib/realtime.ts

@@ -139,7 +139,15 @@ export class RealtimeSession implements VoiceSession {
 
     const dc = pc.createDataChannel("oai-events");
     this.dc = dc;
-    dc.addEventListener("open", () => this.configureSession());
+    // The data channel opens after start() resolves and ICE connects. Fire the
+    // ready/listening signals here, once configureSession has run — not at the
+    // end of start(), which runs before the channel is open.
+    dc.addEventListener("open", () => {
+      this.configureSession();
+      emit({ type: "status", status: "Connected — start talking." });
+      emit({ type: "ready" });
+      emit({ type: "state", state: "listening" });
+    });
     dc.addEventListener("message", (ev) => this.handleEvent(ev.data));
 
     const offer = await pc.createOffer();
@@ -156,9 +164,7 @@ export class RealtimeSession implements VoiceSession {
     });
     if (!sdpResp.ok) throw new Error(`SDP exchange failed: ${sdpResp.status} ${await sdpResp.text()}`);
     await pc.setRemoteDescription({ type: "answer", sdp: await sdpResp.text() });
-
-    emit({ type: "status", status: "Connected — start talking." });
-    emit({ type: "state", state: "listening" });
+    // ready/listening fire from the data channel `open` handler above, not here.
   }
 
   stop(): void {

frontend/lib/voice.ts

@@ -34,6 +34,10 @@ export type VoiceUsage = {
 export type RealtimeEvent =
   | { type: "status"; status: string }
   | { type: "state"; state: VoiceState }
+  // Fired at the genuine ready point — the post-start() handshake completing
+  // (Gemini: setupComplete; OpenAI/Azure: data channel open + configured). The
+  // orb's mic-volume animation gates on this, since start() resolving is not ready.
+  | { type: "ready" }
   | { type: "transcript"; role: "user" | "assistant"; text: string; final: boolean }
   | { type: "tool_call"; name: string; arguments?: unknown; result?: unknown; ok?: boolean }
   | { type: "usage"; usage: VoiceUsage }