Skip to content

Fix degraded AirPods/Bluetooth dictation accuracy #593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rohithgoud30 wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Fix degraded AirPods/Bluetooth dictation accuracy #593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
dfde7ae
Revalidate Core Audio input format and detect Bluetooth route changes
rohithgoud30 Jul 12, 2026
7f9edd6
Add anti-aliased StreamingResampler with unit tests
rohithgoud30 Jul 12, 2026
0d59b54
Wire resampler, format recovery, and readiness gate into ASRService
rohithgoud30 Jul 12, 2026
dd59e59
Install capture-started gate before starting the audio backend
rohithgoud30 Jul 12, 2026
5cad47c
Address review findings on format-change handling and resampler state
rohithgoud30 Jul 12, 2026
219e6e8
Clear format-change flag before reading the format at capture start
rohithgoud30 Jul 12, 2026
c01bcbb
Run session-critical setup unconditionally, gate only the overlay
rohithgoud30 Jul 12, 2026
3d24f0e
Apply dictation prompt configuration before the Private AI prewarm
rohithgoud30 Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */; };
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
5853C8F49B0743C69E9B8C77 /* StreamingResamplerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 900080B1FF6B46458AE79921 /* StreamingResamplerTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
Expand All @@ -38,6 +39,7 @@
7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationE2ETests.swift; sourceTree = "<group>"; };
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
900080B1FF6B46458AE79921 /* StreamingResamplerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamingResamplerTests.swift; sourceTree = "<group>"; };
7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
Expand Down Expand Up @@ -110,6 +112,7 @@
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */,
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
900080B1FF6B46458AE79921 /* StreamingResamplerTests.swift */,
);
path = FluidDictationIntegrationTests;
sourceTree = "<group>";
Expand Down Expand Up @@ -266,6 +269,7 @@
7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */,
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
5853C8F49B0743C69E9B8C77 /* StreamingResamplerTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
204 changes: 201 additions & 3 deletions Sources/CoreAudioCaptureSupport/CoreAudioCaptureSupport.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "include/CoreAudioCaptureSupport.h"

#include <Block.h>
#include <dispatch/dispatch.h>
#include <limits.h>
#include <mach/mach_time.h>
Expand All @@ -22,22 +23,33 @@ typedef struct {

typedef struct {
AudioObjectID deviceID;
AudioStreamID inputStreamID;
AudioDeviceIOProcID ioProcID;
AudioStreamBasicDescription format;
uint32_t bufferFrameSize;
uint32_t bytesPerSample;
bool virtualFormatListenerInstalled;
bool nominalSampleRateListenerInstalled;
dispatch_semaphore_t packetSemaphore;
dispatch_queue_t listenerQueue;
AudioObjectPropertyListenerBlock formatChangedBlock;
_Atomic uint64_t writeIndex;
_Atomic uint64_t readIndex;
_Atomic uint64_t droppedPackets;
_Atomic bool running;
_Atomic bool formatChanged;
FVPacketSlot slots[FV_RING_CAPACITY];
} FVCapture;

static OSStatus fv_get_input_stream_format(
AudioObjectID deviceID,
AudioStreamBasicDescription *format
AudioStreamBasicDescription *format,
AudioStreamID *outStreamID
) {
if (format == NULL) {
return kAudioHardwareBadObjectError;
}

AudioObjectPropertyAddress streamsAddress = {
kAudioDevicePropertyStreams,
kAudioObjectPropertyScopeInput,
Expand Down Expand Up @@ -69,6 +81,9 @@ static OSStatus fv_get_input_stream_format(
if (status != noErr || streamID == kAudioObjectUnknown) {
return status != noErr ? status : kAudioHardwareBadObjectError;
}
if (outStreamID != NULL) {
*outStreamID = streamID;
}

AudioObjectPropertyAddress formatAddress = {
kAudioStreamPropertyVirtualFormat,
Expand Down Expand Up @@ -209,6 +224,10 @@ static OSStatus fv_io_proc(
!atomic_load_explicit(&capture->running, memory_order_relaxed)) {
return noErr;
}
if (atomic_load_explicit(&capture->formatChanged, memory_order_acquire)) {
atomic_fetch_add_explicit(&capture->droppedPackets, 1, memory_order_relaxed);
return noErr;
}

uint32_t frameCount = fv_frame_count(capture, inInputData);
if (frameCount == 0) {
Expand Down Expand Up @@ -282,6 +301,43 @@ static OSStatus fv_io_proc(
return noErr;
}

static bool fv_install_format_listener(
AudioObjectID objectID,
AudioObjectPropertySelector selector,
FVCapture *capture
) {
AudioObjectPropertyAddress address = {
selector,
kAudioObjectPropertyScopeGlobal,
kAudioObjectPropertyElementMain,
};
OSStatus status = AudioObjectAddPropertyListenerBlock(
objectID,
&address,
capture->listenerQueue,
capture->formatChangedBlock
);
return status == noErr;
}

static void fv_remove_format_listener(
AudioObjectID objectID,
AudioObjectPropertySelector selector,
FVCapture *capture
) {
AudioObjectPropertyAddress address = {
selector,
kAudioObjectPropertyScopeGlobal,
kAudioObjectPropertyElementMain,
};
(void) AudioObjectRemovePropertyListenerBlock(
objectID,
&address,
capture->listenerQueue,
capture->formatChangedBlock
);
}

int32_t fv_core_audio_capture_create(
AudioObjectID deviceID,
FVCoreAudioCaptureRef *outCapture
Expand All @@ -297,7 +353,11 @@ int32_t fv_core_audio_capture_create(
}
capture->deviceID = deviceID;

OSStatus status = fv_get_input_stream_format(deviceID, &capture->format);
OSStatus status = fv_get_input_stream_format(
deviceID,
&capture->format,
&capture->inputStreamID
);
if (status == noErr &&
!fv_format_is_supported(&capture->format, &capture->bytesPerSample)) {
status = kAudioHardwareUnsupportedOperationError;
Expand All @@ -324,6 +384,36 @@ int32_t fv_core_audio_capture_create(
atomic_init(&capture->readIndex, 0);
atomic_init(&capture->droppedPackets, 0);
atomic_init(&capture->running, false);
atomic_init(&capture->formatChanged, false);

capture->listenerQueue = dispatch_queue_create(
"com.fluidvoice.audio.format-listener",
DISPATCH_QUEUE_SERIAL
);
if (capture->listenerQueue == NULL) {
#if !OS_OBJECT_USE_OBJC
dispatch_release(capture->packetSemaphore);
#endif
free(capture);
return kAudioHardwareUnspecifiedError;
}
AudioObjectPropertyListenerBlock formatChangedBlock =
^(UInt32 inNumberAddresses, const AudioObjectPropertyAddress *inAddresses) {
(void) inNumberAddresses;
(void) inAddresses;

atomic_store_explicit(&capture->formatChanged, true, memory_order_release);
dispatch_semaphore_signal(capture->packetSemaphore);
};
capture->formatChangedBlock = Block_copy(formatChangedBlock);
if (capture->formatChangedBlock == NULL) {
#if !OS_OBJECT_USE_OBJC
dispatch_release(capture->listenerQueue);
dispatch_release(capture->packetSemaphore);
#endif
free(capture);
return kAudioHardwareUnspecifiedError;
}

status = AudioDeviceCreateIOProcID(
deviceID,
Expand All @@ -332,13 +422,26 @@ int32_t fv_core_audio_capture_create(
&capture->ioProcID
);
if (status != noErr) {
Block_release(capture->formatChangedBlock);
#if !OS_OBJECT_USE_OBJC
dispatch_release(capture->listenerQueue);
dispatch_release(capture->packetSemaphore);
#endif
free(capture);
return status;
}

capture->virtualFormatListenerInstalled = fv_install_format_listener(
capture->inputStreamID,
kAudioStreamPropertyVirtualFormat,
capture
);
capture->nominalSampleRateListenerInstalled = fv_install_format_listener(
capture->deviceID,
kAudioDevicePropertyNominalSampleRate,
capture
);

*outCapture = (FVCoreAudioCaptureRef) capture;
return noErr;
}
Expand All @@ -351,9 +454,79 @@ int32_t fv_core_audio_capture_start(FVCoreAudioCaptureRef captureRef) {
if (atomic_load_explicit(&capture->running, memory_order_acquire)) {
return noErr;
}
dispatch_sync(capture->listenerQueue, ^{});
atomic_store_explicit(&capture->formatChanged, false, memory_order_release);

AudioStreamBasicDescription format;
uint32_t bytesPerSample = 0;
uint32_t bufferFrameSize = 0;
AudioStreamID newStreamID = kAudioObjectUnknown;

OSStatus status = fv_get_input_stream_format(
capture->deviceID,
&format,
&newStreamID
);
if (status == noErr && !fv_format_is_supported(&format, &bytesPerSample)) {
status = kAudioHardwareUnsupportedOperationError;
}
if (status == noErr) {
status = fv_get_buffer_frame_size(capture->deviceID, &bufferFrameSize);
}
if (status == noErr &&
(bufferFrameSize == 0 || bufferFrameSize > FV_MAX_FRAMES_PER_PACKET)) {
status = kAudioHardwareUnsupportedOperationError;
}
if (status != noErr) {
return status;
}

if (newStreamID != capture->inputStreamID) {
if (capture->virtualFormatListenerInstalled) {
fv_remove_format_listener(
capture->inputStreamID,
kAudioStreamPropertyVirtualFormat,
capture
);
capture->virtualFormatListenerInstalled = false;
}
capture->inputStreamID = newStreamID;
capture->virtualFormatListenerInstalled = fv_install_format_listener(
capture->inputStreamID,
kAudioStreamPropertyVirtualFormat,
capture
);
status = fv_get_input_stream_format(
capture->deviceID,
&format,
NULL
);
if (status == noErr && !fv_format_is_supported(&format, &bytesPerSample)) {
status = kAudioHardwareUnsupportedOperationError;
}
if (status != noErr) {
return status;
}
} else if (!capture->virtualFormatListenerInstalled) {
capture->virtualFormatListenerInstalled = fv_install_format_listener(
capture->inputStreamID,
kAudioStreamPropertyVirtualFormat,
capture
);
}
if (!capture->nominalSampleRateListenerInstalled) {
capture->nominalSampleRateListenerInstalled = fv_install_format_listener(
capture->deviceID,
kAudioDevicePropertyNominalSampleRate,
capture
);
}
capture->format = format;
capture->bytesPerSample = bytesPerSample;
capture->bufferFrameSize = bufferFrameSize;

atomic_store_explicit(&capture->running, true, memory_order_release);
OSStatus status = AudioDeviceStart(capture->deviceID, capture->ioProcID);
status = AudioDeviceStart(capture->deviceID, capture->ioProcID);
if (status != noErr) {
atomic_store_explicit(&capture->running, false, memory_order_release);
dispatch_semaphore_signal(capture->packetSemaphore);
Expand Down Expand Up @@ -388,11 +561,30 @@ void fv_core_audio_capture_destroy(FVCoreAudioCaptureRef captureRef) {
if (atomic_load_explicit(&capture->running, memory_order_acquire)) {
(void) fv_core_audio_capture_stop(captureRef);
}
if (capture->virtualFormatListenerInstalled) {
fv_remove_format_listener(
capture->inputStreamID,
kAudioStreamPropertyVirtualFormat,
capture
);
capture->virtualFormatListenerInstalled = false;
}
if (capture->nominalSampleRateListenerInstalled) {
fv_remove_format_listener(
capture->deviceID,
kAudioDevicePropertyNominalSampleRate,
capture
);
capture->nominalSampleRateListenerInstalled = false;
}
if (capture->ioProcID != NULL) {
(void) AudioDeviceDestroyIOProcID(capture->deviceID, capture->ioProcID);
capture->ioProcID = NULL;
}
dispatch_sync(capture->listenerQueue, ^{});
Block_release(capture->formatChangedBlock);
#if !OS_OBJECT_USE_OBJC
dispatch_release(capture->listenerQueue);
dispatch_release(capture->packetSemaphore);
#endif
free(capture);
Expand Down Expand Up @@ -480,6 +672,12 @@ bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef captureRef) {
atomic_load_explicit(&capture->running, memory_order_acquire);
}

bool fv_core_audio_capture_format_changed(FVCoreAudioCaptureRef captureRef) {
FVCapture *capture = (FVCapture *) captureRef;
return capture != NULL &&
atomic_load_explicit(&capture->formatChanged, memory_order_acquire);
}

double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef captureRef) {
const FVCapture *capture = (const FVCapture *) captureRef;
return capture == NULL ? 0.0 : capture->format.mSampleRate;
Expand Down
1 change: 1 addition & 0 deletions Sources/CoreAudioCaptureSupport/include/CoreAudioCaptureSupport.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ void fv_core_audio_capture_clear(FVCoreAudioCaptureRef capture);
void fv_core_audio_capture_wake(FVCoreAudioCaptureRef capture);

bool fv_core_audio_capture_is_running(FVCoreAudioCaptureRef capture);
bool fv_core_audio_capture_format_changed(FVCoreAudioCaptureRef capture);
double fv_core_audio_capture_sample_rate(FVCoreAudioCaptureRef capture);
uint32_t fv_core_audio_capture_buffer_frame_size(FVCoreAudioCaptureRef capture);
uint64_t fv_core_audio_capture_dropped_packet_count(FVCoreAudioCaptureRef capture);
Expand Down
10 changes: 5 additions & 5 deletions Sources/Fluid/ContentView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3147,9 +3147,9 @@ struct ContentView: View {
if shouldPlayStartSound, !self.asr.isRunning {
TranscriptionSoundPlayer.shared.playStartSound()
}
self.captureRecordingContext()
self.prewarmPrivateAIDictationIfNeeded(for: .primary)
await self.asr.start(onCaptureStarted: {
self.captureRecordingContext()
self.prewarmPrivateAIDictationIfNeeded(for: .primary)
if shouldShowDictationOverlay {
self.menuBarManager.showRecordingOverlayImmediately()
}
Expand Down Expand Up @@ -3758,14 +3758,14 @@ extension ContentView {
if SettingsStore.shared.enableTranscriptionSounds, !self.asr.isRunning {
TranscriptionSoundPlayer.shared.playStartSound()
}
self.captureRecordingContext()
self.applyDictationPromptConfiguration(for: SettingsStore.shared.dictationPromptSelection(for: slot))
self.prewarmPrivateAIDictationIfNeeded(for: slot)
await self.asr.start(onCaptureStarted: {
self.captureRecordingContext()
self.applyDictationPromptConfiguration(for: SettingsStore.shared.dictationPromptSelection(for: slot))
self.appBench("overlay_mode_request mode=Dictation")
self.menuBarManager.setOverlayMode(.dictation)
self.menuBarManager.showRecordingOverlayImmediately()
self.appBench("overlay_mode_requested mode=Dictation")
self.prewarmPrivateAIDictationIfNeeded(for: slot)
})
if !self.asr.isRunning {
self.menuBarManager.hideRecordingOverlayImmediately(reason: "asr_start_failed")
Expand Down
Loading
Loading