From f1062115ed619946df323ed9114084258123c09d Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 11:45:39 +0800 Subject: [PATCH 1/7] feat(asr): add SenseVoice language selection type Introduce SettingsStore.SenseVoiceLanguage, the language choice for the SenseVoiceSmall model. - Cases: cantonese (default), mandarin, auto. - embedIndex maps each case to the model's lid_int_dict language embedding input (auto=0, mandarin/zh=3, cantonese/yue=13). - displayName provides localized labels for the settings picker. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit c70bd973dbe7e80d9d552b6715658ddd48bef144) --- .../SettingsStore+SenseVoiceLanguage.swift | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift diff --git a/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift new file mode 100644 index 00000000..6e6dee30 --- /dev/null +++ b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift @@ -0,0 +1,31 @@ +import Foundation + +extension SettingsStore { + /// Language selection for the SenseVoiceSmall model. The raw values are stable + /// persistence keys; `embedIndex` maps to the model's `lid_int_dict` language + /// embedding input (auto = 0, Mandarin/zh = 3, Cantonese/yue = 13). + enum SenseVoiceLanguage: String, CaseIterable, Identifiable, Codable { + case cantonese + case mandarin + case auto + + var id: String { self.rawValue } + + /// SenseVoice `language` encoder input (embed index). + var embedIndex: Int32 { + switch self { + case .auto: return 0 + case .mandarin: return 3 + case .cantonese: return 13 + } + } + + var displayName: String { + switch self { + case .cantonese: return "Cantonese (粵語)" + case .mandarin: return "Mandarin (國語)" + case .auto: return "Auto Detect" + } + } + } +} From 91c0a8d7dbde33cdbef410171ce551fed0049f21 Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 11:45:51 +0800 Subject: [PATCH 2/7] feat(asr): add SenseVoice Small ASR provider (Cantonese) Add SenseVoiceSmall as a selectable on-device ASR model with strong Cantonese accuracy, running on the Apple Neural Engine via CoreML. SenseVoiceProvider.swift (new): - Vendors the 3-stage CoreML pipeline + greedy-CTC decode from FluidInference/FluidAudio (Apache-2.0), since the pinned FluidAudio fork omits the SenseVoice module. Core inference runs in a SenseVoiceEngine actor off the main thread. - Downloads the int8 encoder variant from FluidInference/sensevoice-small-coreml via the app's HuggingFaceModelDownloader, reusing its HTML/markup corrupt-cache detection. - Reads the selected language (SettingsStore.senseVoiceLanguage) per call. - Chunks audio longer than one encoder pass (~100 s cap): splits at the quietest sample near the cap to avoid mid-word cuts, transcribes each chunk, and joins the pieces so long recordings are fully transcribed. - Intel builds get a not-available stub. SettingsStore.swift: - Add the .senseVoiceSmall SpeechModel case with its metadata (display name, download size, accuracy/speed, FunASR provider, brand color) across the exhaustive switches; mark it Apple-Silicon-only and non-streaming. - isInstalled checks the provider's on-disk artifacts on arm64. - Add the persisted senseVoiceLanguage setting (defaults to Cantonese). ASRService.swift: - Route .senseVoiceSmall to a cached SenseVoiceProvider in both the active and download-only provider factories, and clear it on reset. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 18ae6cdf08b91007da61c92a34cbb918cf3d02af) --- Sources/Fluid/Persistence/SettingsStore.swift | 56 ++- Sources/Fluid/Services/ASRService.swift | 16 + .../Fluid/Services/SenseVoiceProvider.swift | 466 ++++++++++++++++++ 3 files changed, 533 insertions(+), 5 deletions(-) create mode 100644 Sources/Fluid/Services/SenseVoiceProvider.swift diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 715b2525..60e5b063 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -3737,6 +3737,7 @@ final class SettingsStore: ObservableObject { case parakeetRealtime = "parakeet-realtime" case qwen3Asr = "qwen3-asr" case cohereTranscribeSixBit = "cohere-transcribe-6bit" + case senseVoiceSmall = "sensevoice-small" case nemotronOffline = "nemotron-3.5-offline" case nemotronStreaming = "nemotron-3.5-streaming" case nemotronStreaming320 = "nemotron-3.5-streaming-320" @@ -3768,6 +3769,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "Parakeet Flash (Beta)" case .qwen3Asr: return "Qwen3 ASR (Beta)" case .cohereTranscribeSixBit: return "Cohere Transcribe" + case .senseVoiceSmall: return "SenseVoice Small" case .nemotronOffline: return "Nemotron 3.5 Multilingual" case .nemotronStreaming: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" case .nemotronStreaming320: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" @@ -3790,6 +3792,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "English Only (Live Streaming)" case .qwen3Asr: return "30 Languages" case .cohereTranscribeSixBit: return "14 Languages (Select Manually)" + case .senseVoiceSmall: return "Cantonese, Mandarin & 50+ (Select Manually)" case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return "Around 40 Languages" case .appleSpeech: return "System Languages" case .appleSpeechAnalyzer: return "EN, ES, FR, DE, IT, JA, KO, PT, ZH" @@ -3805,6 +3808,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "~428.4 MiB" case .qwen3Asr: return "~2.0 GiB" case .cohereTranscribeSixBit: return "~1.54 GiB" + case .senseVoiceSmall: return "~215 MiB" case .nemotronOffline: return "~530.8 MiB" case .nemotronStreaming: return "~668.2 MiB" case .nemotronStreaming320: return "~668.2 MiB" @@ -3826,6 +3830,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 449_190_189 case .qwen3Asr: return 2000 * 1024 * 1024 case .cohereTranscribeSixBit: return 1_650_748_785 + case .senseVoiceSmall: return 225_000_000 case .nemotronOffline: return 556_552_620 case .nemotronStreaming, .nemotronStreaming320: return 700_685_415 case .whisperTiny: return 77_691_713 @@ -3840,14 +3845,14 @@ final class SettingsStore: ObservableObject { var requiresAppleSilicon: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return true + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return true default: return false } } var isWhisperModel: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeech, .appleSpeechAnalyzer: return false + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeech, .appleSpeechAnalyzer: return false default: return true } } @@ -3943,6 +3948,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "Flash Dictation" case .qwen3Asr: return "Qwen3 - Multilingual" case .cohereTranscribeSixBit: return "Cohere - High Accuracy" + case .senseVoiceSmall: return "Cantonese - Fast & Accurate" case .nemotronOffline: return "Nemotron 3.5 Multilingual" case .nemotronStreaming: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" case .nemotronStreaming320: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" @@ -3973,6 +3979,8 @@ final class SettingsStore: ObservableObject { return "Qwen3 multilingual ASR via FluidAudio. Higher quality, heavier memory footprint." case .cohereTranscribeSixBit: return "High-accuracy multilingual transcription. Select the language manually before dictation for best results." + case .senseVoiceSmall: + return "Non-autoregressive FunASR model with state-of-the-art Cantonese accuracy. Runs on the Neural Engine with a tiny memory footprint. Select Cantonese, Mandarin, or auto-detect." case .nemotronOffline: return "Slower but more accurate NVIDIA Nemotron 3.5 transcription. Supports 40 language-locales with auto or manual language selection." case .nemotronStreaming: @@ -4007,6 +4015,8 @@ final class SettingsStore: ObservableObject { return 8.0 case .cohereTranscribeSixBit: return 8.0 + case .senseVoiceSmall: + return 2.0 case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return 8.0 case .appleSpeech, .appleSpeechAnalyzer: @@ -4050,6 +4060,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 5 case .qwen3Asr: return 3 case .cohereTranscribeSixBit: return 3 + case .senseVoiceSmall: return 5 case .nemotronOffline: return 3 case .nemotronStreaming, .nemotronStreaming320: return 4 case .appleSpeech: return 4 @@ -4071,6 +4082,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 4 case .qwen3Asr: return 4 case .cohereTranscribeSixBit: return 5 + case .senseVoiceSmall: return 4 case .nemotronOffline: return 5 case .nemotronStreaming, .nemotronStreaming320: return 4 case .appleSpeech: return 4 @@ -4092,6 +4104,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 1.0 case .qwen3Asr: return 0.45 case .cohereTranscribeSixBit: return 0.85 + case .senseVoiceSmall: return 0.95 case .nemotronOffline: return 0.85 case .nemotronStreaming, .nemotronStreaming320: return 1.0 case .appleSpeech: return 0.60 @@ -4113,6 +4126,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 0.75 case .qwen3Asr: return 0.90 case .cohereTranscribeSixBit: return 0.98 + case .senseVoiceSmall: return 0.90 case .nemotronOffline: return 0.90 case .nemotronStreaming, .nemotronStreaming320: return 0.85 case .appleSpeech: return 0.60 @@ -4143,7 +4157,7 @@ final class SettingsStore: ObservableObject { /// Optimization level for Apple Silicon (for display) var appleSiliconOptimized: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeechAnalyzer: + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeechAnalyzer: return true default: return false @@ -4154,8 +4168,8 @@ final class SettingsStore: ObservableObject { /// Large Whisper models are too slow for streaming, so they only do final transcription on stop. var supportsStreaming: Bool { switch self { - case .qwen3Asr, .whisperMedium, .whisperLargeTurbo, .whisperLarge: - return false // Too slow for real-time chunk processing + case .qwen3Asr, .senseVoiceSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: + return false // Non-streaming / too slow for real-time chunk processing default: return true // All other models support streaming } @@ -4198,6 +4212,7 @@ final class SettingsStore: ObservableObject { case openai = "OpenAI" case qwen = "Qwen" case cohere = "Cohere" + case funasr = "FunASR" } /// Which provider this model belongs to @@ -4211,6 +4226,8 @@ final class SettingsStore: ObservableObject { return .qwen case .cohereTranscribeSixBit: return .cohere + case .senseVoiceSmall: + return .funasr case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: return .openai } @@ -4261,6 +4278,13 @@ final class SettingsStore: ObservableObject { return false } return spec.validatesInstalledArtifacts(at: directory) + case .senseVoiceSmall: + #if arch(arm64) + guard let directory = SenseVoiceProvider.cacheDirectory() else { return false } + return SenseVoiceProvider.artifactsAreComplete(at: directory) + #else + return false + #endif case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: let hint: String switch self { @@ -4338,6 +4362,8 @@ final class SettingsStore: ObservableObject { return "Qwen" case .cohereTranscribeSixBit: return "Cohere" + case .senseVoiceSmall: + return "FunASR" case .appleSpeech, .appleSpeechAnalyzer: return "Apple" case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: @@ -4362,6 +4388,8 @@ final class SettingsStore: ObservableObject { return "#E67E22" case .cohereTranscribeSixBit: return "#FA6B3C" + case .senseVoiceSmall: + return "#FF6A00" // Alibaba Orange case .appleSpeech, .appleSpeechAnalyzer: return "#A2AAAD" // Apple Gray case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: @@ -4557,6 +4585,7 @@ private extension SettingsStore { static let selectedSpeechModel = "SelectedSpeechModel" static let selectedCohereLanguage = "SelectedCohereLanguage" static let selectedNemotronLanguage = "SelectedNemotronLanguage" + static let senseVoiceLanguage = "SenseVoiceLanguage" static let selectedAppleSpeechLocaleIdentifier = "SelectedAppleSpeechLocaleIdentifier" static let externalCoreMLArtifactsDirectories = "ExternalCoreMLArtifactsDirectories" @@ -4696,6 +4725,8 @@ extension SettingsStore.SpeechModel { return "EN" case .cohereTranscribeSixBit: return "AR, DE, EL, EN, ES, FR, IT, JA, KO, NL, PL, PT, VI, ZH" + case .senseVoiceSmall: + return "YUE, ZH, EN, JA, KO + 50 more" case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return "40 language-locales" case .appleSpeechAnalyzer: @@ -4838,6 +4869,21 @@ extension SettingsStore { } } + var senseVoiceLanguage: SenseVoiceLanguage { + get { + if let rawValue = self.defaults.string(forKey: Keys.senseVoiceLanguage), + let language = SenseVoiceLanguage(rawValue: rawValue) + { + return language + } + return .cantonese + } + set { + objectWillChange.send() + self.defaults.set(newValue.rawValue, forKey: Keys.senseVoiceLanguage) + } + } + func externalCoreMLArtifactsDirectory(for model: SpeechModel) -> URL? { guard let spec = model.externalCoreMLSpec else { return nil } let paths = self.defaults.dictionary(forKey: Keys.externalCoreMLArtifactsDirectories) as? [String: String] ?? [:] diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index fbef8c16..60ad6eb2 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -149,6 +149,7 @@ final class ASRService: ObservableObject { private var parakeetRealtimeProvider: ParakeetRealtimeProvider? private var externalCoreMLProvider: ExternalCoreMLTranscriptionProvider? private var nemotronProviders: [NemotronProvider.Mode: NemotronProvider] = [:] + private var senseVoiceProvider: SenseVoiceProvider? private var whisperProvider: WhisperProvider? private var appleSpeechProvider: AppleSpeechProvider? /// Stored as Any? because @available cannot be applied to stored properties @@ -207,6 +208,8 @@ final class ASRService: ObservableObject { return self.getParakeetRealtimeProvider() case .cohereTranscribeSixBit: return self.getExternalCoreMLProvider() + case .senseVoiceSmall: + return self.getSenseVoiceProvider() case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return self.getNemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -259,6 +262,16 @@ final class ASRService: ObservableObject { return provider } + private func getSenseVoiceProvider() -> SenseVoiceProvider { + if let existing = self.senseVoiceProvider { + return existing + } + let provider = SenseVoiceProvider() + self.senseVoiceProvider = provider + DebugLogger.shared.info("ASRService: Created SenseVoice provider", source: "ASRService") + return provider + } + private func getWhisperProvider() -> WhisperProvider { if let existing = whisperProvider { return existing @@ -408,6 +421,8 @@ final class ASRService: ObservableObject { return ParakeetRealtimeProvider() case .cohereTranscribeSixBit: return ExternalCoreMLTranscriptionProvider(modelOverride: model) + case .senseVoiceSmall: + return SenseVoiceProvider(modelOverride: model) case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return NemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -525,6 +540,7 @@ final class ASRService: ObservableObject { self.fluidAudioProvider = nil self.parakeetRealtimeProvider = nil self.externalCoreMLProvider = nil + self.senseVoiceProvider = nil self.whisperProvider = nil self.appleSpeechProvider = nil self._appleSpeechAnalyzerProvider = nil diff --git a/Sources/Fluid/Services/SenseVoiceProvider.swift b/Sources/Fluid/Services/SenseVoiceProvider.swift new file mode 100644 index 00000000..907b9b32 --- /dev/null +++ b/Sources/Fluid/Services/SenseVoiceProvider.swift @@ -0,0 +1,466 @@ +import Foundation + +#if arch(arm64) +@preconcurrency import CoreML + +/// TranscriptionProvider for SenseVoiceSmall (FunASR) — a non-autoregressive +/// multilingual ASR model with strong Cantonese accuracy, running on the Apple +/// Neural Engine via CoreML. +/// +/// The 3-stage CoreML pipeline + greedy-CTC decode is vendored from +/// FluidInference/FluidAudio (Apache-2.0), adapted to FluidVoice's own +/// `HuggingFaceModelDownloader` and settings because the pinned FluidAudio fork +/// does not include the SenseVoice module. Model artifacts come from +/// `FluidInference/sensevoice-small-coreml` (int8 encoder variant). +/// +/// See: https://github.com/FluidInference/FluidAudio (ASR/SenseVoice) +final class SenseVoiceProvider: TranscriptionProvider { + let name = "SenseVoice Small" + var isAvailable: Bool { true } + private(set) var isReady: Bool = false + + // MARK: - Repository / artifacts (int8 encoder) + + private let repositoryOwner = "FluidInference" + private let repositoryName = "sensevoice-small-coreml" + private let repositoryRevision = "main" + + static let preprocessorFile = "SenseVoicePreprocessor.mlmodelc" + static let encoderFile = "SenseVoiceSmall_int8.mlmodelc" + static let vocabularyFile = "vocab.json" + static let requiredItems: [HuggingFaceModelDownloader.ModelItem] = [ + .init(path: preprocessorFile, isDirectory: true), + .init(path: encoderFile, isDirectory: true), + .init(path: vocabularyFile, isDirectory: false), + ] + private static var requiredRelativePaths: [String] { + self.requiredItems.map { $0.path } + } + + private var engine: SenseVoiceEngine? + + // MARK: - Long-audio chunking (samples @ 16 kHz) + + /// Hard cap per encoder pass. The int8 encoder tops out at 1800 LFR frames + /// (~960 samples/frame ≈ 108 s); staying at 100 s keeps a safety margin below + /// the truncation clamp in `SenseVoiceEngine.runEncoder`. + private static let maxChunkSamples = 1_600_000 + /// Never emit a chunk shorter than this (1 s), so a silence-seek can't collapse. + private static let minChunkSamples = 16_000 + /// How far back from the hard cap to hunt for a quiet cut point (2 s). + private static let boundarySearchRadiusSamples = 32_000 + private static let boundaryAnalysisWindowSamples = 1_280 + private static let boundaryAnalysisStrideSamples = 320 + + /// Optional model override (unused today — SenseVoice has a single variant — + /// but kept for parity with the other providers' download plumbing). + var modelOverride: SettingsStore.SpeechModel? + init(modelOverride: SettingsStore.SpeechModel? = nil) { + self.modelOverride = modelOverride + } + + static func cacheDirectory() -> URL? { + FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first? + .appendingPathComponent("sensevoice-small-coreml", isDirectory: true) + } + + static func artifactsAreComplete(at directory: URL) -> Bool { + HuggingFaceModelDownloader.artifactsAreComplete(root: directory, items: self.requiredItems) + } + + func modelsExistOnDisk() -> Bool { + guard let dir = Self.cacheDirectory() else { return false } + return Self.artifactsAreComplete(at: dir) + } + + // MARK: - Preparation + + func prepare(progressHandler: ((ModelPreparationProgress) -> Void)? = nil) async throws { + try Task.checkCancellation() + guard self.isReady == false else { return } + guard let dir = Self.cacheDirectory() else { + throw Self.makeError("Unable to resolve a cache directory for \(self.name).") + } + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + + // Re-sniff present artifacts: a proxy can cache an HTML block page (HTTP 200) in + // place of a model file, so a file-existence check alone would trust a corrupt + // cache forever. Force a re-download when that happens (see #353). + let modelsPresent = self.modelsExistOnDisk() + let cachedArtifactsCorrupt = modelsPresent + && HuggingFaceModelDownloader.cachedPayloadContainsMarkup( + root: dir, relativePaths: Self.requiredRelativePaths) + + if modelsPresent, !cachedArtifactsCorrupt { + DebugLogger.shared.info( + "SenseVoice: artifacts present at \(dir.path); skipping download", + source: "SenseVoice" + ) + } else { + if cachedArtifactsCorrupt { + DebugLogger.shared.warning( + "SenseVoice: cached artifacts at \(dir.path) contain an HTML/markup payload (corrupt); re-downloading", + source: "SenseVoice" + ) + } + DebugLogger.shared.info( + "SenseVoice: artifacts missing; downloading from \(self.repositoryOwner)/\(self.repositoryName)", + source: "SenseVoice" + ) + progressHandler?(.preparingDownload) + let downloader = HuggingFaceModelDownloader( + owner: self.repositoryOwner, + repo: self.repositoryName, + revision: self.repositoryRevision, + requiredItems: Self.requiredItems + ) + try await downloader.ensureModelsPresent(at: dir) { progress, _ in + progressHandler?(.downloading(progress)) + } + try Task.checkCancellation() + guard self.modelsExistOnDisk() else { + throw Self.makeError("SenseVoice artifacts incomplete after download at \(dir.path).") + } + } + + progressHandler?(.optimizing) + try Task.checkCancellation() + let engine = try SenseVoiceEngine( + directory: dir, + preprocessorName: Self.preprocessorFile, + encoderName: Self.encoderFile, + vocabularyFile: Self.vocabularyFile + ) + try Task.checkCancellation() + self.engine = engine + self.isReady = true + progressHandler?(.loading) + DebugLogger.shared.info("SenseVoice: provider ready", source: "SenseVoice") + } + + // MARK: - Transcription + + func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult { + guard let engine = self.engine else { + throw Self.makeError("SenseVoice provider is not ready.") + } + guard samples.isEmpty == false else { + return ASRTranscriptionResult(text: "", confidence: 0) + } + + let language = SettingsStore.shared.senseVoiceLanguage.embedIndex + let startedAt = Date().timeIntervalSince1970 + + let text: String + if samples.count <= Self.maxChunkSamples { + text = try await engine.transcribe(audio: samples, language: language) + } else { + // SenseVoice is non-streaming with a fixed max input; split long audio at + // quiet points and stitch the pieces so nothing past ~108 s is dropped. + text = try await self.transcribeChunked(samples, language: language, engine: engine) + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + self.logBenchmark(samples: samples, text: trimmed, startedAt: startedAt) + return ASRTranscriptionResult(text: text, confidence: trimmed.isEmpty ? 0 : 1) + } + + /// Transcribe audio longer than one encoder pass by cutting it into + /// silence-aligned chunks (≤ `maxChunkSamples`) and joining the results. + private func transcribeChunked( + _ samples: [Float], language: Int32, engine: SenseVoiceEngine + ) async throws -> String { + var pieces: [String] = [] + var offset = 0 + while offset < samples.count { + try Task.checkCancellation() + let end = Self.chunkEnd(in: samples, offset: offset) + let chunk = Array(samples[offset.. Int { + let maxEnd = min(offset + self.maxChunkSamples, samples.count) + guard maxEnd < samples.count else { return samples.count } + + let lowerBound = max(offset + self.minChunkSamples, maxEnd - self.boundarySearchRadiusSamples) + guard lowerBound < maxEnd else { return maxEnd } + return self.quietestBoundary(in: samples, lowerBound: lowerBound, upperBound: maxEnd) ?? maxEnd + } + + /// The sample offset in `[lowerBound, upperBound]` with the lowest local energy, + /// nudged toward `upperBound` to prefer longer chunks on ties. + private static func quietestBoundary(in samples: [Float], lowerBound: Int, upperBound: Int) -> Int? { + let halfWindow = max(1, self.boundaryAnalysisWindowSamples / 2) + let stride = max(1, self.boundaryAnalysisStrideSamples) + var bestBoundary: Int? + var bestScore = Float.greatestFiniteMagnitude + var boundary = lowerBound + + while boundary <= upperBound { + let windowStart = max(0, boundary - halfWindow) + let windowEnd = min(samples.count, boundary + halfWindow) + var energy: Float = 0 + for index in windowStart.. 0 ? Double(elapsedMs) / Double(audioMs) : 0 + DebugLogger.shared.info( + """ + ASR_BENCH provider_final_done model=sensevoice samples=\(samples.count) audioMs=\(audioMs) \ + elapsedMs=\(elapsedMs) textChars=\(text.count) rtf=\(String(format: "%.3f", rtf)) + """, + source: "ASRBenchmark" + ) + } + + private static func makeError(_ description: String) -> NSError { + NSError(domain: "SenseVoiceProvider", code: -1, userInfo: [NSLocalizedDescriptionKey: description]) + } +} + +// MARK: - SenseVoice CoreML engine + +/// Actor that owns the loaded CoreML models and runs the SenseVoiceSmall pipeline +/// off the main thread. Pipeline (vendored from FluidInference/FluidAudio): +/// waveform → [Preprocessor fp32/CPU] → 560-d LFR features → pad to the smallest +/// enumerated bucket → [encoder+CTC int8/ANE] → greedy CTC decode (drop blank 0, +/// collapse repeats) → SentencePiece detokenize → strip `<|...|>` meta tags. +actor SenseVoiceEngine { + /// Configuration constants mirroring the `FluidInference/sensevoice-small-coreml` conversion. + private enum Config { + static let featureDim = 560 + static let buckets = [128, 256, 512, 1024, 1800] + static let numQueryTokens = 4 + static let blankId = 0 + static let defaultTextNorm: Int32 = 15 // woitn (no inverse text-norm) + static let waveformScale: Float = 32_768.0 + static let sentencePieceWordBoundary = "\u{2581}" // "▁" + static var maxFrames: Int { buckets.last ?? 1800 } + static func pickBucket(forFrames frames: Int) -> Int { + for b in buckets where b >= frames { return b } + return buckets.last ?? 1800 + } + } + + private let preprocessor: MLModel + private let encoder: MLModel + private let vocabulary: [Int: String] + + init(directory: URL, preprocessorName: String, encoderName: String, vocabularyFile: String) throws { + // Preprocessor must run fp32 on CPU (power-spectrum/log exceed fp16 range, + // and its big identity convs fail ANE compile). The int8 encoder runs on ANE. + let cpuConfig = MLModelConfiguration() + cpuConfig.computeUnits = .cpuOnly + let aneConfig = MLModelConfiguration() + aneConfig.computeUnits = .cpuAndNeuralEngine + + self.preprocessor = try MLModel( + contentsOf: directory.appendingPathComponent(preprocessorName), configuration: cpuConfig) + self.encoder = try MLModel( + contentsOf: directory.appendingPathComponent(encoderName), configuration: aneConfig) + self.vocabulary = try Self.loadVocabulary( + from: directory.appendingPathComponent(vocabularyFile)) + } + + /// Transcribe 16 kHz mono float samples (in [-1, 1]) for the given language embed index. + func transcribe(audio: [Float], language: Int32) throws -> String { + let features = try self.runPreprocessor(audio: audio) + let (logits, validFrames) = try self.runEncoder(features: features, language: language) + return self.decode(logits: logits, validFrames: validFrames) + } + + // MARK: - Pipeline + + private func runPreprocessor(audio: [Float]) throws -> MLMultiArray { + let n = audio.count + let waveform = try MLMultiArray(shape: [1, n as NSNumber], dataType: .float32) + let wptr = waveform.dataPointer.assumingMemoryBound(to: Float32.self) + let scale = Config.waveformScale + for i in 0.. (MLMultiArray, Int) { + let dim = Config.featureDim + var t = features.shape[1].intValue + // Clamp to the largest enumerated bucket (~108 s). Longer audio is truncated; + // callers needing full coverage should chunk before this point. + if t > Config.maxFrames { + t = Config.maxFrames + } + let bucket = Config.pickBucket(forFrames: t) + + // Zero-padded [1, bucket, 560] with the first T feature frames copied in. + let speech = try MLMultiArray(shape: [1, bucket as NSNumber, dim as NSNumber], dataType: .float32) + let sptr = speech.dataPointer.assumingMemoryBound(to: Float32.self) + memset(sptr, 0, bucket * dim * MemoryLayout.size) + let count = t * dim + if features.dataType == .float32 { + memcpy(sptr, features.dataPointer, count * MemoryLayout.size) + } else { + for i in 0..<|emo|><|event|><|itn|>` tags. + private func decode(logits: MLMultiArray, validFrames: Int) -> String { + let vocab = logits.shape[2].intValue + let frames = min(validFrames, logits.shape[1].intValue) + var ids: [Int] = [] + var prev = -1 + + func appendArgmax(frameBase: (Int) -> Float) { + var best = 0 + var bestVal = frameBase(0) + for v in 1.. bestVal { + bestVal = x + best = v + } + } + if best != Config.blankId && best != prev { ids.append(best) } + prev = best + } + + if logits.dataType == .float32 { + let p = logits.dataPointer.assumingMemoryBound(to: Float32.self) + for t in 0..", with: "", options: .regularExpression) + .trimmingCharacters(in: .whitespaces) + } + + // MARK: - Loading + + private static func loadVocabulary(from url: URL) throws -> [Int: String] { + let data = try Data(contentsOf: url) + // Canonical format: JSON array ["", "", ...]. + if let arr = try? JSONSerialization.jsonObject(with: data) as? [String] { + var v: [Int: String] = [:] + v.reserveCapacity(arr.count) + for (i, tok) in arr.enumerated() { v[i] = tok } + return v + } + // Fallback: {"0": "", ...} dictionary. + if let dict = try? JSONSerialization.jsonObject(with: data) as? [String: String] { + var v: [Int: String] = [:] + v.reserveCapacity(dict.count) + for (k, tok) in dict { if let i = Int(k) { v[i] = tok } } + return v + } + throw Self.makeError("Failed to parse \(url.lastPathComponent) (expected array or dict)") + } + + private static func makeError(_ description: String) -> NSError { + NSError(domain: "SenseVoiceEngine", code: -1, userInfo: [NSLocalizedDescriptionKey: description]) + } +} + +#else + +/// Intel stub — SenseVoice's int8 encoder targets the Apple Neural Engine. +final class SenseVoiceProvider: TranscriptionProvider { + let name = "SenseVoice Small (Apple Silicon ONLY)" + var isAvailable: Bool { false } + private(set) var isReady: Bool = false + + init(modelOverride: SettingsStore.SpeechModel? = nil) {} + + func prepare(progressHandler: ((ModelPreparationProgress) -> Void)?) async throws { + throw NSError( + domain: "SenseVoiceProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "SenseVoice requires Apple Silicon."] + ) + } + + func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult { + throw NSError( + domain: "SenseVoiceProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "SenseVoice requires Apple Silicon."] + ) + } + + func modelsExistOnDisk() -> Bool { false } +} + +#endif From 1bba9393c72cd12d041f333f5f79bde9fb451bdf Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 11:45:57 +0800 Subject: [PATCH 3/7] feat(asr): add SenseVoice language picker to voice settings Show a language menu (Cantonese / Mandarin / Auto Detect) in the voice engine settings when the SenseVoice Small model is selected, mirroring the existing per-model language chip. Selecting a language updates SettingsStore.senseVoiceLanguage. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 0d8d67e6080303dede2fea6ae25526f9bc24cd9a) --- .../UI/AISettingsView+SpeechRecognition.swift | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift index 269c633e..3493d01f 100644 --- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift +++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift @@ -569,6 +569,25 @@ extension VoiceEngineSettingsView { .buttonStyle(.plain) } else if model == .nemotronOffline || model == .nemotronStreaming || model == .nemotronStreaming320 { self.nemotronLanguagePickerButton + } else if model == .senseVoiceSmall { + Menu { + ForEach(SettingsStore.SenseVoiceLanguage.allCases) { language in + Button { + guard language != self.settings.senseVoiceLanguage else { return } + self.settings.senseVoiceLanguage = language + } label: { + HStack { + Text(language.displayName) + if language == self.settings.senseVoiceLanguage { + Image(systemName: "checkmark") + } + } + } + } + } label: { + self.languageChipLabel(self.settings.senseVoiceLanguage.displayName) + } + .buttonStyle(.plain) } } From cc4bc4d870447673fe955cb090ce706f431fee02 Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 12:02:56 +0800 Subject: [PATCH 4/7] fix(asr): accept compiled .mlmodelc bundles without metadata.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit artifactIsComplete required metadata.json for every .mlmodelc, but valid compiled bundles can omit it — the SenseVoice export ships coremldata.bin + model.mil + weights/weight.bin and no metadata.json. That made ensureModelsPresent throw "artifacts incomplete" after a fully successful download and left SpeechModel.isInstalled permanently false. Require only the load-critical files (coremldata.bin + weights/weight.bin); metadata.json and model.mil are descriptive and now optional. No regression for existing bundles (Parakeet etc.), which already contain both required files. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 38e6eea47af64d122235eb4e485e47d09172fdc8) --- Sources/Fluid/Networking/ModelDownloader.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Sources/Fluid/Networking/ModelDownloader.swift b/Sources/Fluid/Networking/ModelDownloader.swift index a7922f3f..f8ad47e7 100644 --- a/Sources/Fluid/Networking/ModelDownloader.swift +++ b/Sources/Fluid/Networking/ModelDownloader.swift @@ -365,10 +365,11 @@ final class HuggingFaceModelDownloader { } } if url.pathExtension == "mlmodelc" { - // `model.mil` is optional in valid compiled bundles (the Flash preprocessor ships - // without it), so installation truth comes from compiled metadata plus weights. + // `model.mil` and `metadata.json` are descriptive and absent from some valid + // compiled bundles (the Flash preprocessor omits `model.mil`; the SenseVoice + // export omits `metadata.json`). Load-critical truth is the compiled index plus + // weights, so require only those two. return self.fileHasContents(at: url.appendingPathComponent("coremldata.bin")) - && self.fileHasContents(at: url.appendingPathComponent("metadata.json")) && self.fileHasContents( at: url .appendingPathComponent("weights", isDirectory: true) From e146adf3653f970da22b10bfd104de200b8064ea Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 12:02:56 +0800 Subject: [PATCH 5/7] fix(asr): cap SenseVoice chunks to the preprocessor's 30s input limit The CoreML preprocessor constrains its waveform input to 3200..480000 samples (0.2..30 s), rejecting anything longer. The chunk cap was 1,600,000 samples (~100 s), aimed at the encoder's ~108 s frame limit, so any recording over 30 s crashed with an out-of-range input error. - Lower maxChunkSamples to 460,000 (28.75 s), safely below 480,000; the single-pass path now only ever sees sub-limit audio and long recordings chunk correctly. - Zero-pad waveforms below the 3200-sample floor so very short recordings and small trailing chunks don't trip the same input constraint. Verified end-to-end against the int8 CoreML model: 28 s single-pass and a 181 s recording (7 chunks) both transcribe fully with no crash. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit dcfbb23e347f13af5934a68ebd357c85c5e145d4) --- Sources/Fluid/Services/SenseVoiceProvider.swift | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Sources/Fluid/Services/SenseVoiceProvider.swift b/Sources/Fluid/Services/SenseVoiceProvider.swift index 907b9b32..19bd8756 100644 --- a/Sources/Fluid/Services/SenseVoiceProvider.swift +++ b/Sources/Fluid/Services/SenseVoiceProvider.swift @@ -41,10 +41,10 @@ final class SenseVoiceProvider: TranscriptionProvider { // MARK: - Long-audio chunking (samples @ 16 kHz) - /// Hard cap per encoder pass. The int8 encoder tops out at 1800 LFR frames - /// (~960 samples/frame ≈ 108 s); staying at 100 s keeps a safety margin below - /// the truncation clamp in `SenseVoiceEngine.runEncoder`. - private static let maxChunkSamples = 1_600_000 + /// Hard cap per pass. The CoreML preprocessor's waveform input is constrained to + /// 3200…480000 samples (0.2…30 s); anything longer is rejected outright, so this + /// sits safely below 480000 rather than at the encoder's ~108 s frame limit. + private static let maxChunkSamples = 460_000 /// Never emit a chunk shorter than this (1 s), so a silence-seek can't collapse. private static let minChunkSamples = 16_000 /// How far back from the hard cap to hunt for a quiet cut point (2 s). @@ -273,6 +273,8 @@ actor SenseVoiceEngine { static let defaultTextNorm: Int32 = 15 // woitn (no inverse text-norm) static let waveformScale: Float = 32_768.0 static let sentencePieceWordBoundary = "\u{2581}" // "▁" + /// Preprocessor waveform input floor (0.2 s); shorter audio is zero-padded up to it. + static let minWaveformSamples = 3_200 static var maxFrames: Int { buckets.last ?? 1800 } static func pickBucket(forFrames frames: Int) -> Int { for b in buckets where b >= frames { return b } @@ -310,11 +312,14 @@ actor SenseVoiceEngine { // MARK: - Pipeline private func runPreprocessor(audio: [Float]) throws -> MLMultiArray { - let n = audio.count + // The preprocessor rejects waveforms below its 3200-sample (0.2 s) floor, so a very + // short recording or a small trailing chunk is zero-padded (silence) up to it. + let n = max(audio.count, Config.minWaveformSamples) let waveform = try MLMultiArray(shape: [1, n as NSNumber], dataType: .float32) let wptr = waveform.dataPointer.assumingMemoryBound(to: Float32.self) + memset(wptr, 0, n * MemoryLayout.size) let scale = Config.waveformScale - for i in 0.. Date: Sat, 11 Jul 2026 14:43:34 +0800 Subject: [PATCH 6/7] fix(asr): correct SenseVoice Cantonese language index (yue = 7, not 13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cantonese embedIndex was 13, but 13 is the model's <|nospeech|> control token; the correct <|yue|> embedding index is 7. Verified against the model's vocab.json (token <|yue|> maps to embed index 7 in lid_int_dict). Feeding 13 still transcribed clear Cantonese because the acoustic model dominates for clean audio, but it passed "nospeech" as the language hint — wrong semantics that could suppress or degrade output on noisy/borderline speech. Mandarin (3) and auto (0) were already correct. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit efeb959e70c3537e5a3e9d4ca2c3b212410a071a) --- .../Persistence/SettingsStore+SenseVoiceLanguage.swift | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift index 6e6dee30..3b679dc0 100644 --- a/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift +++ b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift @@ -3,7 +3,9 @@ import Foundation extension SettingsStore { /// Language selection for the SenseVoiceSmall model. The raw values are stable /// persistence keys; `embedIndex` maps to the model's `lid_int_dict` language - /// embedding input (auto = 0, Mandarin/zh = 3, Cantonese/yue = 13). + /// embedding input (auto = 0, Mandarin/zh = 3, Cantonese/yue = 7). Verified + /// against vocab.json: token `<|yue|>` → embed index 7 (index 13 is + /// `<|nospeech|>`, not Cantonese). enum SenseVoiceLanguage: String, CaseIterable, Identifiable, Codable { case cantonese case mandarin @@ -16,7 +18,7 @@ extension SettingsStore { switch self { case .auto: return 0 case .mandarin: return 3 - case .cantonese: return 13 + case .cantonese: return 7 } } From 16ce3cd550310269741d9319e5f565e4738387da Mon Sep 17 00:00:00 2001 From: Kelvin Ng Date: Sat, 11 Jul 2026 17:12:46 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix(asr):=20address=20SenseVoice=20review?= =?UTF-8?q?=20=E2=80=94=20CJK=20chunk=20join,=20trimmed=20result,=20pinned?= =?UTF-8?q?=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve three review findings on the SenseVoice transcription path: - Chunk join no longer inserts a literal space at every silence cut. A content-aware `joinChunks` adds a separator only at Latin-script boundaries; CJK (Cantonese/Mandarin/Japanese/Korean) chunks are concatenated directly, so long recordings no longer produce output like `喺度 做嘢`. - transcribe() returns the trimmed text (not the raw string), matching the confidence check and the other providers' contract. - Pin the model download to a specific commit (cdea3526163035c19915d4a10268992d018ebd46) instead of `main`, so an upstream re-upload can't silently ship a different/incompatible encoder to fresh installs. Co-Authored-By: Claude Opus 4.8 --- .../Fluid/Services/SenseVoiceProvider.swift | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/Sources/Fluid/Services/SenseVoiceProvider.swift b/Sources/Fluid/Services/SenseVoiceProvider.swift index 19bd8756..325ef248 100644 --- a/Sources/Fluid/Services/SenseVoiceProvider.swift +++ b/Sources/Fluid/Services/SenseVoiceProvider.swift @@ -23,7 +23,10 @@ final class SenseVoiceProvider: TranscriptionProvider { private let repositoryOwner = "FluidInference" private let repositoryName = "sensevoice-small-coreml" - private let repositoryRevision = "main" + // Pinned to a specific commit for reproducibility: tracking `main` would let an + // upstream re-upload silently deliver a different (possibly incompatible) encoder to + // fresh installs. Bump this deliberately when adopting a new model revision. + private let repositoryRevision = "cdea3526163035c19915d4a10268992d018ebd46" static let preprocessorFile = "SenseVoicePreprocessor.mlmodelc" static let encoderFile = "SenseVoiceSmall_int8.mlmodelc" @@ -155,14 +158,16 @@ final class SenseVoiceProvider: TranscriptionProvider { if samples.count <= Self.maxChunkSamples { text = try await engine.transcribe(audio: samples, language: language) } else { - // SenseVoice is non-streaming with a fixed max input; split long audio at - // quiet points and stitch the pieces so nothing past ~108 s is dropped. + // SenseVoice is non-streaming with a fixed ~30 s max input; split long audio + // at quiet points and stitch the pieces so nothing past the cap is dropped. text = try await self.transcribeChunked(samples, language: language, engine: engine) } let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) self.logBenchmark(samples: samples, text: trimmed, startedAt: startedAt) - return ASRTranscriptionResult(text: text, confidence: trimmed.isEmpty ? 0 : 1) + // Return the trimmed text so downstream consumers get the same clean string used + // for the confidence check — matching the other providers' contract. + return ASRTranscriptionResult(text: trimmed, confidence: trimmed.isEmpty ? 0 : 1) } /// Transcribe audio longer than one encoder pass by cutting it into @@ -183,7 +188,42 @@ final class SenseVoiceProvider: TranscriptionProvider { } offset = end } - return pieces.joined(separator: " ") + return Self.joinChunks(pieces) + } + + /// Join chunk transcripts, inserting a space **only** at Latin-script boundaries. + /// SentencePiece already spaces words within a chunk (via `▁`); CJK text has no + /// inter-character spaces, so a blanket `" "` separator would inject a spurious space + /// at every silence cut (e.g. `喺度 做嘢`). Decide per boundary from the adjacent + /// characters, which also handles the auto-detect case correctly. + static func joinChunks(_ pieces: [String]) -> String { + var result = "" + for piece in pieces where piece.isEmpty == false { + if let last = result.unicodeScalars.last, + let first = piece.unicodeScalars.first, + !isCJKScalar(last), !isCJKScalar(first) + { + result += " " + } + result += piece + } + return result + } + + /// Whether a scalar belongs to a script written without inter-word spaces + /// (CJK ideographs + Kana + Hangul), used to pick the chunk-join separator. + static func isCJKScalar(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.value { + case 0x3040...0x30FF, // Hiragana, Katakana + 0x3400...0x4DBF, // CJK Extension A + 0x4E00...0x9FFF, // CJK Unified Ideographs + 0xAC00...0xD7A3, // Hangul syllables + 0xF900...0xFAFF, // CJK Compatibility Ideographs + 0x20000...0x2FA1F: // CJK Extensions B–F + Supplement + return true + default: + return false + } } /// End index (exclusive) of the chunk starting at `offset`. For a non-final