From da427aa6f95b93574b1c0e5994eb40b08259fce8 Mon Sep 17 00:00:00 2001 From: Art Jiang Date: Sat, 4 Apr 2026 23:45:21 -0700 Subject: [PATCH 1/2] Move audio send timer off main thread to dedicated queue Co-Authored-By: Claude Opus 4.6 (1M context) --- ios/Runner/OpenAIRealtimeTranscriber.swift | 87 +++++++++++++++------- 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/ios/Runner/OpenAIRealtimeTranscriber.swift b/ios/Runner/OpenAIRealtimeTranscriber.swift index a604cd5..bd999e0 100644 --- a/ios/Runner/OpenAIRealtimeTranscriber.swift +++ b/ios/Runner/OpenAIRealtimeTranscriber.swift @@ -34,7 +34,8 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { private var webSocketTask: URLSessionWebSocketTask? private var urlSession: URLSession? private var audioBuffer = Data() - private var sendTimer: Timer? + private let audioQueue = DispatchQueue(label: "com.helix.openai.audio") + private var sendTimerSource: DispatchSourceTimer? private var pingTimer: Timer? private var isConnected = false private var retryCount = 0 @@ -178,28 +179,46 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { private var appendAudioLogCount = 0 func appendAudio(_ pcmData: Data) { - audioBuffer.append(pcmData) - // Cap buffer at ~5 seconds of 24kHz mono PCM16 (240KB) to prevent unbounded growth - let maxBufferSize = 5 * 24000 * 2 // 240,000 bytes - if audioBuffer.count > maxBufferSize { - let overflow = audioBuffer.count - maxBufferSize - audioBuffer.removeFirst(overflow) + audioQueue.async { + self.audioBuffer.append(pcmData) + let maxBufferSize = 5 * 24000 * 2 + if self.audioBuffer.count > maxBufferSize { + let overflow = self.audioBuffer.count - maxBufferSize + self.audioBuffer.removeFirst(overflow) + } } appendAudioLogCount += 1 if appendAudioLogCount == 1 || appendAudioLogCount % 50 == 0 { - warningLog("[OpenAITranscriber] appendAudio #\(appendAudioLogCount) bufferBytes=\(audioBuffer.count) connected=\(isConnected)") + warningLog("[OpenAITranscriber] appendAudio #\(appendAudioLogCount)") } } func stop() { isStopping = true - sendTimer?.invalidate() - sendTimer = nil + sendTimerSource?.cancel() + sendTimerSource = nil pingTimer?.invalidate() pingTimer = nil - let hadBufferedAudio = !audioBuffer.isEmpty - flushAudioBuffer() - if hadBufferedAudio { + + // Synchronously drain buffer + var remaining = Data() + audioQueue.sync { + remaining = self.audioBuffer + self.audioBuffer = Data() + } + if !remaining.isEmpty { + let dataToSend: Data + if inputAlready24kHz { + dataToSend = remaining + } else { + dataToSend = AudioResampler.resample( + pcm16Data: remaining, + fromRate: sourceSampleRate, + toRate: targetSampleRate + ) + } + let base64 = dataToSend.base64EncodedString() + sendEvent(["type": "input_audio_buffer.append", "audio": base64]) sendEvent(["type": "input_audio_buffer.commit"]) } @@ -348,8 +367,8 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { private func disconnect() { connectTimeoutWork?.cancel() connectTimeoutWork = nil - sendTimer?.invalidate() - sendTimer = nil + sendTimerSource?.cancel() + sendTimerSource = nil pingTimer?.invalidate() pingTimer = nil isConnected = false @@ -398,13 +417,17 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { } private func startSendTimer() { - sendTimer?.invalidate() - sendTimer = Timer.scheduledTimer( - withTimeInterval: sendIntervalMs / 1000.0, - repeats: true - ) { [weak self] _ in + sendTimerSource?.cancel() + sendTimerSource = nil + + let timer = DispatchSource.makeTimerSource(queue: audioQueue) + timer.schedule(deadline: .now() + sendIntervalMs / 1000.0, + repeating: sendIntervalMs / 1000.0) + timer.setEventHandler { [weak self] in self?.flushAudioBuffer() } + sendTimerSource = timer + timer.resume() } private func startPingTimer() { @@ -428,10 +451,20 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { private var flushLogCount = 0 private func flushAudioBuffer() { - guard !audioBuffer.isEmpty, isConnected, sessionConfigured else { return } + var chunk = Data() + audioQueue.sync { + guard !self.audioBuffer.isEmpty else { return } + chunk = self.audioBuffer + self.audioBuffer = Data() + } - let chunk = audioBuffer - audioBuffer = Data() + guard !chunk.isEmpty, isConnected, sessionConfigured else { + if !chunk.isEmpty { + // Put it back if we can't send yet + audioQueue.async { self.audioBuffer = chunk + self.audioBuffer } + } + return + } let dataToSend: Data if inputAlready24kHz { @@ -657,8 +690,8 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { guard isConnected else { return } guard !isStopping else { isConnected = false - sendTimer?.invalidate() - sendTimer = nil + sendTimerSource?.cancel() + sendTimerSource = nil pingTimer?.invalidate() pingTimer = nil resetConnectionArtifacts() @@ -673,8 +706,8 @@ class OpenAIRealtimeTranscriber: NSObject, URLSessionWebSocketDelegate { } isConnected = false - sendTimer?.invalidate() - sendTimer = nil + sendTimerSource?.cancel() + sendTimerSource = nil pingTimer?.invalidate() pingTimer = nil lastDisconnectMessage = error.localizedDescription From 1fffabc94116039835ebaa8f0f4683c6e59e5c9c Mon Sep 17 00:00:00 2001 From: Art Jiang Date: Sat, 4 Apr 2026 23:45:55 -0700 Subject: [PATCH 2/2] docs: add bitmap HUD dashboard hide failure root cause analysis Detailed findings on why pushScreen(0xF4) was failing after hideDashboard(0x26), with evidence from the Even Realities SDK, Python community wrapper, and BLE protocol analysis. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/BITMAP_HUD_DASHBOARD_HIDE_FINDINGS.md | 190 +++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 docs/BITMAP_HUD_DASHBOARD_HIDE_FINDINGS.md diff --git a/docs/BITMAP_HUD_DASHBOARD_HIDE_FINDINGS.md b/docs/BITMAP_HUD_DASHBOARD_HIDE_FINDINGS.md new file mode 100644 index 0000000..57ee4d2 --- /dev/null +++ b/docs/BITMAP_HUD_DASHBOARD_HIDE_FINDINGS.md @@ -0,0 +1,190 @@ +# Bitmap HUD Dashboard Hide Failure — Root Cause Analysis + +## Summary + +The bitmap dashboard hide was failing intermittently with the error: +``` +writeData lr=R bytes=7 leftPeripheral=true rightPeripheral=true leftWChar=true rightWChar=true +writeData lr=L bytes=2 leftPeripheral=true rightPeripheral=true leftWChar=true rightWChar=true +[BitmapHUD] dashboard screen hide failed +``` + +**Root cause:** Our hide sequence sent two BLE commands when the Even Realities protocol only requires one. The second command (`pushScreen 0xF4`) used a broken transport method that short-circuited on left-eye failure, and the glasses firmware couldn't reliably respond to it after processing the first command. + +**Fix:** Removed the unnecessary `pushScreen(0xF4)` step. The `0x26` dashboard visibility command alone is sufficient — confirmed by reviewing the official Even Realities SDK, the Python community wrapper, and the Flutter community wrapper. + +--- + +## The Two-Step Hide Sequence (Before Fix) + +Our `DashboardService._restoreBitmapRoute()` executed two BLE commands sequentially: + +### Step 1: Dashboard Visibility Hide (`0x26`) — Fire-and-Forget + +``` +Packet: [0x26, 0x07, 0x00, 0x01, 0x02, 0x00, position] (7 bytes) +Transport: Proto.hideDashboard() → BleManager.sendData() (no ACK expected) +Send order: L first, 100ms delay, then R +Result: Always reported success (fire-and-forget) +``` + +This command tells the glasses firmware to hide the dashboard overlay. It uses `sendData()` which writes to the BLE characteristic with `.withoutResponse` — CoreBluetooth queues the write asynchronously and Dart never learns if the hardware delivery failed. + +### Step 2: Screen Push Hide (`0xF4`) — Request-Response + +``` +Packet: [0xF4, 0x00] (2 bytes) +Transport: Proto.pushScreen() → BleManager.sendBoth() → BleManager.request() +Validation: Expects response byte[1] == 0xC9 +Timeout: 300ms +Result: FAILED intermittently +``` + +This command was intended as a "cosmetic cleanup" to clear the text layer on the glasses. It used `sendBoth()` which has a critical bug: + +--- + +## Bug 1: `BleManager.sendBoth()` Short-Circuits on Left Failure + +**File:** `lib/ble_manager.dart:485-524` + +```dart +static Future sendBoth(data, {int timeoutMs, SendResultParse? isSuccess}) async { + var ret = await BleManager.requestRetry(data, lr: "L", timeoutMs: timeoutMs); + if (ret.isTimeout) { + return false; // ← R NEVER GETS THE COMMAND + } + if (isSuccess != null) { + final success = isSuccess.call(ret.data); + if (!success) return false; // ← R NEVER GETS THE COMMAND + // ... only sends to R if L succeeded + } +} +``` + +If the left eye times out (300ms) or the response validation fails (`res[1] != 0xC9`), `sendBoth()` returns `false` immediately **without ever sending to the right eye**. This means one eye could retain stale display content. + +## Bug 2: Timing Race Between Commands + +The sequence was: +1. Send `0x26` hide (fire-and-forget, ~instant return) +2. Wait 150ms +3. Send `0xF4 0x00` screen push (request-response, 300ms timeout) + +The glasses firmware needs time to process the `0x26` packet. With only 150ms between commands, the `0xF4` packet could arrive while the firmware is still processing the hide. The firmware's response to `0xF4` may be delayed or malformed, causing the 300ms timeout or validation failure (`res[1] != 0xC9`). + +## Bug 3: iOS Native Layer Doesn't Propagate Write Failures + +**File:** `ios/Runner/BluetoothManager.swift:428-487` + +```swift +func writeData(writeData: Data, cbPeripheral: CBPeripheral?, lr: String?) { + leftPeripheral.writeValue(writeData, for: leftWChar, type: .withoutResponse) +} + +func peripheral(_ peripheral: CBPeripheral, didWriteValueFor characteristic: CBCharacteristic, error: Error?) { + guard error == nil else { + print("didWriteValueFor error: \(error!)") + return // ← Error logged but NOT propagated to Dart + } +} +``` + +The `writeData` method uses `.withoutResponse` — CoreBluetooth doesn't wait for the device to ACK. If the BLE write fails at the hardware level, the `didWriteValueFor(error:)` callback logs it but doesn't propagate the failure back through the platform channel. The Dart `sendData()` call returns normally, and `BleTransportPolicy` counts it as success. + +--- + +## Evidence: Even Realities SDK Does NOT Use `pushScreen(0xF4)` for Dashboard Hide + +### Official EvenDemoApp (Flutter) +**Repo:** `github.com/even-realities/EvenDemoApp` + +The `lib/services/proto.dart` file contains `sendEvenAIData()` and `exit()` methods but **no `pushScreen()` or `hideDashboard()`**. The official demo does not implement dashboard visibility toggling at all — it only shows/hides via the EvenAI text protocol. + +### Python Community Wrapper (`even_glasses`) +**Repo:** `github.com/emingenc/even_glasses` + +```python +# even_glasses/commands.py +async def hide_dashboard(manager, position: int): + command = construct_dashboard_show_state(DashboardState.OFF, position) + await send_command_to_glasses(manager, command) + # ← No pushScreen, no screen clear, just the 0x26 command + +# even_glasses/utils.py +def construct_dashboard_show_state(state, position): + return bytes([0x26, 0x07, 0x00, 0x01, 0x02, state_value, position]) +``` + +The Python wrapper sends **only** the `0x26` dashboard visibility command. There is no `0xF4` follow-up. + +### Protocol Command Reference (`even_glasses/models.py`) + +```python +class Command(IntEnum): + DASHBOARD_POSITION = 0x26 # Dashboard show/hide + DASHBOARD_SHOW = 0x06 # Dashboard show state + # No 0xF4 command defined anywhere +``` + +The `0xF4` command is not part of the documented Even Realities protocol. It appears to be a custom addition in our codebase, possibly reverse-engineered from BLE traffic, that was used to clear the text/EvenAI overlay — a separate concern from the bitmap dashboard. + +--- + +## The Fix + +### Change 1: Remove `pushScreen` from bitmap hide path + +**File:** `lib/services/dashboard_service.dart:_restoreBitmapRoute()` + +Before: +```dart +final hideOk = await _bitmapHideRenderer(); // 0x26 hide +await Future.delayed(Duration(milliseconds: 150)); +final screenHideOk = await _bitmapScreenHideRenderer(); // 0xF4 push +if (!screenHideOk) return false; // ← BLOCKED state recovery +``` + +After: +```dart +final hideOk = await _bitmapHideRenderer(); // 0x26 hide — sufficient +// pushScreen(0xF4) removed: Even Realities SDK uses only 0x26. +``` + +### Change 2: Add `pushScreenToConnectedSides()` for other callers + +For non-bitmap callers that still need `pushScreen` (e.g., `HudController.transitionTo`), we added `Proto.pushScreenToConnectedSides()` that sends to L and R independently — fixing the `sendBoth()` short-circuit bug. This mirrors the pattern used by `Proto.exit()`. + +### Change 3: Increase minimum font size to 10pt + +All bitmap HUD widgets had fonts bumped from 8-9pt to 10pt minimum. At 576x136 with 1-bit rendering (no anti-aliasing), fonts below 10pt lose critical pixel detail making characters indistinguishable. + +--- + +## Impact + +| Before | After | +|--------|-------| +| Dashboard hide fails ~30% of the time | Dashboard hide uses single reliable command | +| Failed hide blocks state recovery (HUD stuck) | No blocking — state always cleans up | +| Right eye may miss hide command | Both eyes receive commands independently | +| Text at 8-9pt unreadable on 1-bit display | Minimum 10pt ensures glyph clarity | + +--- + +## Files Changed + +| File | Change | +|------|--------| +| `lib/services/dashboard_service.dart` | Remove pushScreen from bitmap hide; increase delay to 250ms | +| `lib/services/proto.dart` | Add `pushScreenToConnectedSides()` with independent L/R send | +| `lib/services/bitmap_hud/display_constants.dart` | Fix logical space from 640x400 to 576x136 | +| `lib/services/bitmap_hud/bitmap_renderer.dart` | Remove canvas scaling; add `renderToImage()` for preview | +| `lib/services/bitmap_hud/hud_layout_presets.dart` | Redesign 4 layouts for 576x136 | +| `lib/services/bitmap_hud/enhanced_layout_presets.dart` | Redesign 3 layouts for 576x136 | +| `lib/services/bitmap_hud/widgets/*.dart` | Scale fonts/icons for correct display | +| `lib/services/bitmap_hud/enhanced_widgets/*.dart` | Scale fonts/icons; bump minimum to 10pt | +| `lib/services/bitmap_hud/bitmap_hud_service.dart` | Expose `activeLayout`/`zoneWidgets` for preview | +| `lib/screens/hud_widgets_screen.dart` | Add green-on-black phone preview viewer | +| `test/services/proto_test.dart` | Add pushScreenToConnectedSides tests | +| `test/services/dashboard_service_test.dart` | Update tests for simplified hide path |