From 7c27596873f739071004bfec104e6c1147c0ac0b Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Sun, 5 Apr 2026 12:23:47 +0300 Subject: [PATCH 1/7] feat: add TURN-TCP support and E2E tests for WebRTC fallback Add TURN-over-TCP ICE server alongside existing STUN for networks where UDP is blocked. ICE naturally prefers direct UDP and falls back to TURN-TCP relay via coturn. Includes Playwright E2E tests verifying both relay-only and dual-mode paths against local k8s slim-bit-invert. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/package.json | 1 + .../sdk/src/realtime/webrtc-connection.ts | 5 +- packages/sdk/tests/e2e-turn-tcp.test.ts | 167 ++++++++++++++++++ packages/sdk/vitest.config.e2e-turn-tcp.ts | 18 ++ packages/sdk/vitest.config.ts | 2 +- 5 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 packages/sdk/tests/e2e-turn-tcp.test.ts create mode 100644 packages/sdk/vitest.config.e2e-turn-tcp.ts diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 44af5de7..4f257a8f 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -35,6 +35,7 @@ "test": "vitest unit", "test:e2e": "vitest e2e", "test:e2e:realtime": "vitest --config vitest.config.e2e-realtime.ts", + "test:e2e:turn-tcp": "vitest --config vitest.config.e2e-turn-tcp.ts", "typecheck": "tsc --noEmit", "format": "biome format --write", "format:check": "biome check", diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index dc5802b9..eb5bb91e 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -13,7 +13,10 @@ import type { SetImageAckMessage, } from "./types"; -const ICE_SERVERS: RTCIceServer[] = [{ urls: "stun:stun.l.google.com:19302" }]; +const ICE_SERVERS: RTCIceServer[] = [ + { urls: "stun:stun.l.google.com:19302" }, + { urls: "turn:127.0.0.1:3478?transport=tcp", username: "turn", credential: "turn" }, +]; const AVATAR_SETUP_TIMEOUT_MS = 30_000; // 30 seconds interface ConnectionCallbacks { diff --git a/packages/sdk/tests/e2e-turn-tcp.test.ts b/packages/sdk/tests/e2e-turn-tcp.test.ts new file mode 100644 index 00000000..c9c141df --- /dev/null +++ b/packages/sdk/tests/e2e-turn-tcp.test.ts @@ -0,0 +1,167 @@ +declare const __DECART_API_KEY__: string; +declare const __WEBRTC_BASE_URL__: string; + +import { + createDecartClient, + type CustomModelDefinition, + type DecartSDKError, + type SelectedCandidatePairEvent, +} from "@decartai/sdk"; +import { beforeAll, describe, expect, it } from "vitest"; + +function createSyntheticStream(fps: number, width: number, height: number): MediaStream { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas.captureStream(fps); +} + +const BIT_INVERT_MODEL: CustomModelDefinition = { + name: "bit_invert", + urlPath: "/ws", + fps: 25, + width: 512, + height: 512, +}; + +const TIMEOUT = 2 * 60 * 1000; // 2 minutes + +/** + * Wraps the global RTCPeerConnection so every new instance uses + * the given iceTransportPolicy. Returns a cleanup function to restore. + */ +function overrideIceTransportPolicy(policy: RTCIceTransportPolicy): () => void { + const OriginalPC = globalThis.RTCPeerConnection; + globalThis.RTCPeerConnection = class extends OriginalPC { + constructor(config?: RTCConfiguration) { + super({ ...config, iceTransportPolicy: policy }); + } + } as typeof RTCPeerConnection; + return () => { + globalThis.RTCPeerConnection = OriginalPC; + }; +} + +/** + * Collects the selectedCandidatePair diagnostic from a realtime client. + * The event is buffered during connect() and flushed via setTimeout(0) + * after connect() resolves, so registering immediately catches it. + */ +function collectSelectedCandidatePair( + realtimeClient: { on: (event: "diagnostic", handler: (e: { name: string; data: unknown }) => void) => void }, +): Promise { + return new Promise((resolve) => { + const handler = (event: { name: string; data: unknown }) => { + if (event.name === "selectedCandidatePair") { + resolve(event.data as SelectedCandidatePairEvent); + } + }; + realtimeClient.on("diagnostic", handler); + // Fallback: if the event was already emitted before we registered, resolve after a delay + setTimeout(() => resolve(null), 5000); + }); +} + +describe("TURN-TCP E2E Tests", { timeout: TIMEOUT, retry: 2 }, () => { + let apiKey: string; + let webrtcBaseUrl: string; + + beforeAll(() => { + apiKey = __DECART_API_KEY__; + webrtcBaseUrl = __WEBRTC_BASE_URL__; + if (!apiKey) { + throw new Error( + "DECART_API_KEY environment variable not set. Run with: DECART_API_KEY=your_key pnpm test:e2e:turn-tcp", + ); + } + if (!webrtcBaseUrl) { + throw new Error( + "WEBRTC_BASE_URL environment variable not set. Set it to your local k8s WebSocket URL.", + ); + } + }); + + it("TURN-TCP relay only (iceTransportPolicy=relay)", async () => { + const restore = overrideIceTransportPolicy("relay"); + + try { + const client = createDecartClient({ apiKey, realtimeBaseUrl: webrtcBaseUrl }); + const stream = createSyntheticStream(BIT_INVERT_MODEL.fps, BIT_INVERT_MODEL.width, BIT_INVERT_MODEL.height); + + let remoteStreamReceived = false; + + const realtimeClient = await client.realtime.connect(stream, { + model: BIT_INVERT_MODEL, + onRemoteStream: () => { + remoteStreamReceived = true; + }, + }); + + // Register diagnostic listener immediately - buffered events flush on next macrotask + const candidatePairPromise = collectSelectedCandidatePair(realtimeClient); + + const errors: DecartSDKError[] = []; + realtimeClient.on("error", (err) => errors.push(err)); + + try { + expect(["connected", "generating"]).toContain(realtimeClient.getConnectionState()); + expect(realtimeClient.sessionId).toBeTruthy(); + expect(remoteStreamReceived).toBe(true); + expect(errors).toEqual([]); + + // With relay-only policy, the selected candidate must be a relay (TURN) + const pair = await candidatePairPromise; + if (pair) { + expect(pair.local.candidateType).toBe("relay"); + } + } finally { + realtimeClient.disconnect(); + for (const track of stream.getTracks()) track.stop(); + } + + expect(realtimeClient.getConnectionState()).toBe("disconnected"); + } finally { + restore(); + } + }); + + it("Both UDP + TURN available (default iceTransportPolicy=all)", async () => { + const client = createDecartClient({ apiKey, realtimeBaseUrl: webrtcBaseUrl }); + const stream = createSyntheticStream(BIT_INVERT_MODEL.fps, BIT_INVERT_MODEL.width, BIT_INVERT_MODEL.height); + + let remoteStreamReceived = false; + + const realtimeClient = await client.realtime.connect(stream, { + model: BIT_INVERT_MODEL, + onRemoteStream: () => { + remoteStreamReceived = true; + }, + }); + + // Register diagnostic listener immediately + const candidatePairPromise = collectSelectedCandidatePair(realtimeClient); + + const errors: DecartSDKError[] = []; + realtimeClient.on("error", (err) => errors.push(err)); + + try { + expect(["connected", "generating"]).toContain(realtimeClient.getConnectionState()); + expect(realtimeClient.sessionId).toBeTruthy(); + expect(remoteStreamReceived).toBe(true); + expect(errors).toEqual([]); + + // With default policy, ICE should prefer direct UDP over relay. + // In Docker/NAT environments, the local candidate may appear as "prflx" + // (peer-reflexive) rather than "host", but it should NOT be "relay". + const pair = await candidatePairPromise; + if (pair) { + expect(pair.local.candidateType).not.toBe("relay"); + } + } finally { + realtimeClient.disconnect(); + for (const track of stream.getTracks()) track.stop(); + } + + expect(realtimeClient.getConnectionState()).toBe("disconnected"); + }); +}); diff --git a/packages/sdk/vitest.config.e2e-turn-tcp.ts b/packages/sdk/vitest.config.e2e-turn-tcp.ts new file mode 100644 index 00000000..6ebc477e --- /dev/null +++ b/packages/sdk/vitest.config.e2e-turn-tcp.ts @@ -0,0 +1,18 @@ +import { playwright } from "@vitest/browser-playwright"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + define: { + __DECART_API_KEY__: JSON.stringify(process.env.DECART_API_KEY), + __WEBRTC_BASE_URL__: JSON.stringify(process.env.WEBRTC_BASE_URL || "wss://slim-bit-invert.dev.localhost"), + }, + test: { + include: ["tests/e2e-turn-tcp.test.ts"], + browser: { + enabled: true, + provider: playwright(), + headless: true, + instances: [{ browser: "chromium" }], + }, + }, +}); diff --git a/packages/sdk/vitest.config.ts b/packages/sdk/vitest.config.ts index 79aba294..3e9ef0d6 100644 --- a/packages/sdk/vitest.config.ts +++ b/packages/sdk/vitest.config.ts @@ -2,6 +2,6 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - exclude: ["tests/e2e-realtime.test.ts"], + exclude: ["tests/e2e-realtime.test.ts", "tests/e2e-turn-tcp.test.ts"], }, }); From 99e6b84c2a4a8073606399b58c7be0ba81550da0 Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Mon, 6 Apr 2026 01:17:30 +0300 Subject: [PATCH 2/7] feat: accept server-driven TURN config via signaling - Handle `turn_config` WebSocket message from server to receive TURN servers dynamically (for production server-side control) - Add `iceServers` option to connect() for manual/internal testing - Remove hardcoded TURN from ICE_SERVERS (now server-driven or explicit) - Skip relay-only E2E test until server-side aioice TURN allocation is verified Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/src/realtime/client.ts | 2 ++ packages/sdk/src/realtime/types.ts | 10 ++++++++- .../sdk/src/realtime/webrtc-connection.ts | 22 ++++++++++++++----- packages/sdk/src/realtime/webrtc-manager.ts | 2 ++ packages/sdk/tests/e2e-turn-tcp.test.ts | 11 +++++++++- 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index f1869b0a..7298bbec 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -96,6 +96,7 @@ const realTimeClientConnectOptionsSchema = z.object({ }); export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; + iceServers?: RTCIceServer[]; }; export type Events = { @@ -194,6 +195,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { modelName: options.model.name, initialImage, initialPrompt, + iceServers: options.iceServers, }); const manager = webrtcManager; diff --git a/packages/sdk/src/realtime/types.ts b/packages/sdk/src/realtime/types.ts index e1618e8e..6f61c32c 100644 --- a/packages/sdk/src/realtime/types.ts +++ b/packages/sdk/src/realtime/types.ts @@ -71,6 +71,13 @@ export type SessionIdMessage = { server_port: number; }; +export type TurnConfigMessage = { + type: "turn_config"; + urls: string[]; + username: string; + credential: string; +}; + export type ConnectionState = "connecting" | "connected" | "generating" | "disconnected" | "reconnecting"; // Incoming message types (from server) @@ -85,7 +92,8 @@ export type IncomingWebRTCMessage = | GenerationStartedMessage | GenerationTickMessage | GenerationEndedMessage - | SessionIdMessage; + | SessionIdMessage + | TurnConfigMessage; // Outgoing message types (to server) export type OutgoingWebRTCMessage = diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index eb5bb91e..b3671133 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -11,12 +11,10 @@ import type { PromptAckMessage, SessionIdMessage, SetImageAckMessage, + TurnConfigMessage, } from "./types"; -const ICE_SERVERS: RTCIceServer[] = [ - { urls: "stun:stun.l.google.com:19302" }, - { urls: "turn:127.0.0.1:3478?transport=tcp", username: "turn", credential: "turn" }, -]; +const DEFAULT_ICE_SERVERS: RTCIceServer[] = [{ urls: "stun:stun.l.google.com:19302" }]; const AVATAR_SETUP_TIMEOUT_MS = 30_000; // 30 seconds interface ConnectionCallbacks { @@ -31,6 +29,7 @@ interface ConnectionCallbacks { initialPrompt?: { text: string; enhance?: boolean }; logger?: Logger; onDiagnostic?: DiagnosticEmitter; + iceServers?: RTCIceServer[]; } type WsMessageEvents = { @@ -38,6 +37,7 @@ type WsMessageEvents = { setImageAck: SetImageAckMessage; sessionId: SessionIdMessage; generationTick: GenerationTickMessage; + turnConfig: TurnConfigMessage; }; const noopDiagnostic: DiagnosticEmitter = () => {}; @@ -49,6 +49,7 @@ export class WebRTCConnection { private connectionReject: ((error: Error) => void) | null = null; private logger: Logger; private emitDiagnostic: DiagnosticEmitter; + private turnServers: RTCIceServer[] = []; state: ConnectionState = "disconnected"; websocketMessagesEmitter = mitt(); constructor(private callbacks: ConnectionCallbacks = {}) { @@ -257,6 +258,12 @@ export class WebRTCConnection { return; } + if (msg.type === "turn_config") { + this.turnServers = [{ urls: msg.urls, username: msg.username, credential: msg.credential }]; + this.websocketMessagesEmitter.emit("turnConfig", msg); + return; + } + // All other messages require peer connection if (!this.pc) return; @@ -414,7 +421,11 @@ export class WebRTCConnection { }); this.pc.close(); } - this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS }); + const iceServers: RTCIceServer[] = [ + ...(this.callbacks.iceServers ?? DEFAULT_ICE_SERVERS), + ...this.turnServers, + ]; + this.pc = new RTCPeerConnection({ iceServers }); this.setState("connecting"); if (this.localStream) { @@ -560,6 +571,7 @@ export class WebRTCConnection { this.ws?.close(); this.ws = null; this.localStream = null; + this.turnServers = []; this.setState("disconnected"); } diff --git a/packages/sdk/src/realtime/webrtc-manager.ts b/packages/sdk/src/realtime/webrtc-manager.ts index 71408fb7..7bcd6613 100644 --- a/packages/sdk/src/realtime/webrtc-manager.ts +++ b/packages/sdk/src/realtime/webrtc-manager.ts @@ -19,6 +19,7 @@ export interface WebRTCConfig { modelName?: string; initialImage?: string; initialPrompt?: { text: string; enhance?: boolean }; + iceServers?: RTCIceServer[]; } const PERMANENT_ERRORS = [ @@ -66,6 +67,7 @@ export class WebRTCManager { initialPrompt: config.initialPrompt, logger: this.logger, onDiagnostic: config.onDiagnostic, + iceServers: config.iceServers, }); } diff --git a/packages/sdk/tests/e2e-turn-tcp.test.ts b/packages/sdk/tests/e2e-turn-tcp.test.ts index c9c141df..1cc9a21d 100644 --- a/packages/sdk/tests/e2e-turn-tcp.test.ts +++ b/packages/sdk/tests/e2e-turn-tcp.test.ts @@ -24,6 +24,11 @@ const BIT_INVERT_MODEL: CustomModelDefinition = { height: 512, }; +const TURN_ICE_SERVERS: RTCIceServer[] = [ + { urls: "stun:stun.l.google.com:19302" }, + { urls: "turn:127.0.0.1:3478?transport=tcp", username: "turn", credential: "turn" }, +]; + const TIMEOUT = 2 * 60 * 1000; // 2 minutes /** @@ -81,7 +86,9 @@ describe("TURN-TCP E2E Tests", { timeout: TIMEOUT, retry: 2 }, () => { } }); - it("TURN-TCP relay only (iceTransportPolicy=relay)", async () => { + // Requires server-side aioice TURN-TCP allocation to work (server must produce relay candidates). + // Skip until server-side TURN candidate generation is verified. + it.skip("TURN-TCP relay only (iceTransportPolicy=relay)", async () => { const restore = overrideIceTransportPolicy("relay"); try { @@ -95,6 +102,7 @@ describe("TURN-TCP E2E Tests", { timeout: TIMEOUT, retry: 2 }, () => { onRemoteStream: () => { remoteStreamReceived = true; }, + iceServers: TURN_ICE_SERVERS, }); // Register diagnostic listener immediately - buffered events flush on next macrotask @@ -136,6 +144,7 @@ describe("TURN-TCP E2E Tests", { timeout: TIMEOUT, retry: 2 }, () => { onRemoteStream: () => { remoteStreamReceived = true; }, + iceServers: TURN_ICE_SERVERS, }); // Register diagnostic listener immediately From 1bec2aac4c8abba2cb88842e1a553c29e4938025 Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Mon, 6 Apr 2026 16:52:03 +0300 Subject: [PATCH 3/7] fix: ICE restart when turn_config arrives after PC creation When the server sends turn_config after the PeerConnection is already created (race: bouncer acks Phase 2 locally before upstream pump delivers turn_config), update the PC's ICE servers via setConfiguration() and trigger an ICE restart so TURN candidates are gathered. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/src/realtime/webrtc-connection.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index b3671133..cbd9e733 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -261,6 +261,21 @@ export class WebRTCConnection { if (msg.type === "turn_config") { this.turnServers = [{ urls: msg.urls, username: msg.username, credential: msg.credential }]; this.websocketMessagesEmitter.emit("turnConfig", msg); + + // If PC already exists (turn_config arrived after Phase 3 started), + // update ICE servers and restart ICE to pick up TURN. + if (this.pc) { + const iceServers: RTCIceServer[] = [ + ...(this.callbacks.iceServers ?? DEFAULT_ICE_SERVERS), + ...this.turnServers, + ]; + this.pc.setConfiguration({ iceServers }); + const offer = await this.pc.createOffer({ iceRestart: true }); + this.modifyVP8Bitrate(offer); + await this.callbacks.customizeOffer?.(offer); + await this.pc.setLocalDescription(offer); + this.send({ type: "offer", sdp: offer.sdp || "" }); + } return; } From 0d2be202b15e5a94727b2c00610c0a5e740c2f80 Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Mon, 6 Apr 2026 17:04:11 +0300 Subject: [PATCH 4/7] fix: ignore stale answers after ICE restart When turn_config triggers an ICE restart, the answer to the original offer may arrive after the new offer resets signaling state to stable. Guard setRemoteDescription(answer) with a signalingState check to drop stale answers. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/src/realtime/webrtc-connection.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index cbd9e733..589c69fa 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -300,6 +300,11 @@ export class WebRTCConnection { break; } case "answer": + // Ignore stale answers (e.g. from pre-ICE-restart offer) + if (this.pc.signalingState !== "have-local-offer") { + this.logger.debug("Ignoring stale answer", { signalingState: this.pc.signalingState }); + break; + } await this.pc.setRemoteDescription({ type: "answer", sdp: msg.sdp, From 9348290826f1195ab134aec627efde2a6da2cd5c Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Mon, 6 Apr 2026 17:19:02 +0300 Subject: [PATCH 5/7] fix: wait for turn_config before creating PeerConnection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace ICE restart approach with a Phase 2.5 wait: after Phase 2 completes, wait up to 2s for turn_config to arrive before creating the PeerConnection. This ensures TURN servers are included from the start — one offer, one answer, no re-negotiation. Also reverts the stale answer guard (no longer needed without ICE restart). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../sdk/src/realtime/webrtc-connection.ts | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index 589c69fa..48c1ecc3 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -163,6 +163,22 @@ export class WebRTCConnection { }); } + // Phase 2.5: Wait for turn_config if not yet received. + // turn_config arrives from the server during Phase 2 but may race with + // set_image_ack. Wait briefly so setupNewPeerConnection() includes TURN servers. + if (this.turnServers.length === 0) { + await Promise.race([ + new Promise((resolve) => { + const handler = () => resolve(); + this.websocketMessagesEmitter.on("turnConfig", handler); + // Clean up if resolved by timeout + connectAbort.catch(() => this.websocketMessagesEmitter.off("turnConfig", handler)); + }), + new Promise((resolve) => setTimeout(resolve, 2000)), + connectAbort, + ]); + } + // Phase 3: WebRTC handshake const handshakeStart = performance.now(); await this.setupNewPeerConnection(); @@ -261,21 +277,6 @@ export class WebRTCConnection { if (msg.type === "turn_config") { this.turnServers = [{ urls: msg.urls, username: msg.username, credential: msg.credential }]; this.websocketMessagesEmitter.emit("turnConfig", msg); - - // If PC already exists (turn_config arrived after Phase 3 started), - // update ICE servers and restart ICE to pick up TURN. - if (this.pc) { - const iceServers: RTCIceServer[] = [ - ...(this.callbacks.iceServers ?? DEFAULT_ICE_SERVERS), - ...this.turnServers, - ]; - this.pc.setConfiguration({ iceServers }); - const offer = await this.pc.createOffer({ iceRestart: true }); - this.modifyVP8Bitrate(offer); - await this.callbacks.customizeOffer?.(offer); - await this.pc.setLocalDescription(offer); - this.send({ type: "offer", sdp: offer.sdp || "" }); - } return; } @@ -300,11 +301,6 @@ export class WebRTCConnection { break; } case "answer": - // Ignore stale answers (e.g. from pre-ICE-restart offer) - if (this.pc.signalingState !== "have-local-offer") { - this.logger.debug("Ignoring stale answer", { signalingState: this.pc.signalingState }); - break; - } await this.pc.setRemoteDescription({ type: "answer", sdp: msg.sdp, From d5cd0bea98f0f768858474961c895e6f5da668c7 Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Mon, 6 Apr 2026 22:07:53 +0300 Subject: [PATCH 6/7] feat: add iceTransportPolicy connect option Forwards ice_transport_policy query param to the server, which filters TURN URLs by transport (tcp/udp/all) and configures server-side ICE accordingly. Usage: client.realtime.connect(stream, { model, iceTransportPolicy: "tcp" }) Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/src/realtime/client.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index 7298bbec..f173c000 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -97,6 +97,7 @@ const realTimeClientConnectOptionsSchema = z.object({ export type RealTimeClientConnectOptions = Omit, "model"> & { model: ModelDefinition | CustomModelDefinition; iceServers?: RTCIceServer[]; + iceTransportPolicy?: "tcp" | "udp" | "all"; }; export type Events = { @@ -173,7 +174,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer(); webrtcManager = new WebRTCManager({ - webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}`, + webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}${options.iceTransportPolicy ? `&ice_transport_policy=${encodeURIComponent(options.iceTransportPolicy)}` : ""}`, integration, logger, onDiagnostic: (name, data) => { From 9bebe2d8a54aae43ccf2618aa605facde3540be4 Mon Sep 17 00:00:00 2001 From: nagar-decart Date: Tue, 7 Apr 2026 11:38:05 +0300 Subject: [PATCH 7/7] fix: skip turn_config wait when TURN not requested, fix handler leak - Only wait for turn_config in Phase 2.5 when iceTransportPolicy is set (server expected to send TURN config). Connections without TURN have zero added latency. - Clean up turnConfig event handler after Promise.race resolves, preventing leaked handlers on reconnection cycles. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/sdk/src/realtime/client.ts | 1 + packages/sdk/src/realtime/webrtc-connection.ts | 15 ++++++++------- packages/sdk/src/realtime/webrtc-manager.ts | 2 ++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index f173c000..47f0b4e3 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -197,6 +197,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialImage, initialPrompt, iceServers: options.iceServers, + expectTurnConfig: !!options.iceTransportPolicy, }); const manager = webrtcManager; diff --git a/packages/sdk/src/realtime/webrtc-connection.ts b/packages/sdk/src/realtime/webrtc-connection.ts index 48c1ecc3..e17e6023 100644 --- a/packages/sdk/src/realtime/webrtc-connection.ts +++ b/packages/sdk/src/realtime/webrtc-connection.ts @@ -30,6 +30,7 @@ interface ConnectionCallbacks { logger?: Logger; onDiagnostic?: DiagnosticEmitter; iceServers?: RTCIceServer[]; + expectTurnConfig?: boolean; } type WsMessageEvents = { @@ -164,19 +165,19 @@ export class WebRTCConnection { } // Phase 2.5: Wait for turn_config if not yet received. - // turn_config arrives from the server during Phase 2 but may race with - // set_image_ack. Wait briefly so setupNewPeerConnection() includes TURN servers. - if (this.turnServers.length === 0) { + // Only wait when the server is expected to send TURN config (iceTransportPolicy was set). + // turn_config arrives during Phase 2 but may race with set_image_ack. + if (this.callbacks.expectTurnConfig && this.turnServers.length === 0) { + let turnHandler: (() => void) | null = null; await Promise.race([ new Promise((resolve) => { - const handler = () => resolve(); - this.websocketMessagesEmitter.on("turnConfig", handler); - // Clean up if resolved by timeout - connectAbort.catch(() => this.websocketMessagesEmitter.off("turnConfig", handler)); + turnHandler = () => resolve(); + this.websocketMessagesEmitter.on("turnConfig", turnHandler); }), new Promise((resolve) => setTimeout(resolve, 2000)), connectAbort, ]); + if (turnHandler) this.websocketMessagesEmitter.off("turnConfig", turnHandler); } // Phase 3: WebRTC handshake diff --git a/packages/sdk/src/realtime/webrtc-manager.ts b/packages/sdk/src/realtime/webrtc-manager.ts index 7bcd6613..d00ec5af 100644 --- a/packages/sdk/src/realtime/webrtc-manager.ts +++ b/packages/sdk/src/realtime/webrtc-manager.ts @@ -20,6 +20,7 @@ export interface WebRTCConfig { initialImage?: string; initialPrompt?: { text: string; enhance?: boolean }; iceServers?: RTCIceServer[]; + expectTurnConfig?: boolean; } const PERMANENT_ERRORS = [ @@ -68,6 +69,7 @@ export class WebRTCManager { logger: this.logger, onDiagnostic: config.onDiagnostic, iceServers: config.iceServers, + expectTurnConfig: config.expectTurnConfig, }); }