diff --git a/Sources/Fluid/Networking/ModelDownloader.swift b/Sources/Fluid/Networking/ModelDownloader.swift index a7922f3f..f8ad47e7 100644 --- a/Sources/Fluid/Networking/ModelDownloader.swift +++ b/Sources/Fluid/Networking/ModelDownloader.swift @@ -365,10 +365,11 @@ final class HuggingFaceModelDownloader { } } if url.pathExtension == "mlmodelc" { - // `model.mil` is optional in valid compiled bundles (the Flash preprocessor ships - // without it), so installation truth comes from compiled metadata plus weights. + // `model.mil` and `metadata.json` are descriptive and absent from some valid + // compiled bundles (the Flash preprocessor omits `model.mil`; the SenseVoice + // export omits `metadata.json`). Load-critical truth is the compiled index plus + // weights, so require only those two. return self.fileHasContents(at: url.appendingPathComponent("coremldata.bin")) - && self.fileHasContents(at: url.appendingPathComponent("metadata.json")) && self.fileHasContents( at: url .appendingPathComponent("weights", isDirectory: true) diff --git a/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift new file mode 100644 index 00000000..3b679dc0 --- /dev/null +++ b/Sources/Fluid/Persistence/SettingsStore+SenseVoiceLanguage.swift @@ -0,0 +1,33 @@ +import Foundation + +extension SettingsStore { + /// Language selection for the SenseVoiceSmall model. The raw values are stable + /// persistence keys; `embedIndex` maps to the model's `lid_int_dict` language + /// embedding input (auto = 0, Mandarin/zh = 3, Cantonese/yue = 7). Verified + /// against vocab.json: token `<|yue|>` → embed index 7 (index 13 is + /// `<|nospeech|>`, not Cantonese). + enum SenseVoiceLanguage: String, CaseIterable, Identifiable, Codable { + case cantonese + case mandarin + case auto + + var id: String { self.rawValue } + + /// SenseVoice `language` encoder input (embed index). + var embedIndex: Int32 { + switch self { + case .auto: return 0 + case .mandarin: return 3 + case .cantonese: return 7 + } + } + + var displayName: String { + switch self { + case .cantonese: return "Cantonese (粵語)" + case .mandarin: return "Mandarin (國語)" + case .auto: return "Auto Detect" + } + } + } +} diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 715b2525..60e5b063 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -3737,6 +3737,7 @@ final class SettingsStore: ObservableObject { case parakeetRealtime = "parakeet-realtime" case qwen3Asr = "qwen3-asr" case cohereTranscribeSixBit = "cohere-transcribe-6bit" + case senseVoiceSmall = "sensevoice-small" case nemotronOffline = "nemotron-3.5-offline" case nemotronStreaming = "nemotron-3.5-streaming" case nemotronStreaming320 = "nemotron-3.5-streaming-320" @@ -3768,6 +3769,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "Parakeet Flash (Beta)" case .qwen3Asr: return "Qwen3 ASR (Beta)" case .cohereTranscribeSixBit: return "Cohere Transcribe" + case .senseVoiceSmall: return "SenseVoice Small" case .nemotronOffline: return "Nemotron 3.5 Multilingual" case .nemotronStreaming: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" case .nemotronStreaming320: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" @@ -3790,6 +3792,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "English Only (Live Streaming)" case .qwen3Asr: return "30 Languages" case .cohereTranscribeSixBit: return "14 Languages (Select Manually)" + case .senseVoiceSmall: return "Cantonese, Mandarin & 50+ (Select Manually)" case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return "Around 40 Languages" case .appleSpeech: return "System Languages" case .appleSpeechAnalyzer: return "EN, ES, FR, DE, IT, JA, KO, PT, ZH" @@ -3805,6 +3808,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "~428.4 MiB" case .qwen3Asr: return "~2.0 GiB" case .cohereTranscribeSixBit: return "~1.54 GiB" + case .senseVoiceSmall: return "~215 MiB" case .nemotronOffline: return "~530.8 MiB" case .nemotronStreaming: return "~668.2 MiB" case .nemotronStreaming320: return "~668.2 MiB" @@ -3826,6 +3830,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 449_190_189 case .qwen3Asr: return 2000 * 1024 * 1024 case .cohereTranscribeSixBit: return 1_650_748_785 + case .senseVoiceSmall: return 225_000_000 case .nemotronOffline: return 556_552_620 case .nemotronStreaming, .nemotronStreaming320: return 700_685_415 case .whisperTiny: return 77_691_713 @@ -3840,14 +3845,14 @@ final class SettingsStore: ObservableObject { var requiresAppleSilicon: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return true + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return true default: return false } } var isWhisperModel: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeech, .appleSpeechAnalyzer: return false + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeech, .appleSpeechAnalyzer: return false default: return true } } @@ -3943,6 +3948,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return "Flash Dictation" case .qwen3Asr: return "Qwen3 - Multilingual" case .cohereTranscribeSixBit: return "Cohere - High Accuracy" + case .senseVoiceSmall: return "Cantonese - Fast & Accurate" case .nemotronOffline: return "Nemotron 3.5 Multilingual" case .nemotronStreaming: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" case .nemotronStreaming320: return "Nemotron Speech 3.5 - Ultra Fast Low Latency" @@ -3973,6 +3979,8 @@ final class SettingsStore: ObservableObject { return "Qwen3 multilingual ASR via FluidAudio. Higher quality, heavier memory footprint." case .cohereTranscribeSixBit: return "High-accuracy multilingual transcription. Select the language manually before dictation for best results." + case .senseVoiceSmall: + return "Non-autoregressive FunASR model with state-of-the-art Cantonese accuracy. Runs on the Neural Engine with a tiny memory footprint. Select Cantonese, Mandarin, or auto-detect." case .nemotronOffline: return "Slower but more accurate NVIDIA Nemotron 3.5 transcription. Supports 40 language-locales with auto or manual language selection." case .nemotronStreaming: @@ -4007,6 +4015,8 @@ final class SettingsStore: ObservableObject { return 8.0 case .cohereTranscribeSixBit: return 8.0 + case .senseVoiceSmall: + return 2.0 case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return 8.0 case .appleSpeech, .appleSpeechAnalyzer: @@ -4050,6 +4060,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 5 case .qwen3Asr: return 3 case .cohereTranscribeSixBit: return 3 + case .senseVoiceSmall: return 5 case .nemotronOffline: return 3 case .nemotronStreaming, .nemotronStreaming320: return 4 case .appleSpeech: return 4 @@ -4071,6 +4082,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 4 case .qwen3Asr: return 4 case .cohereTranscribeSixBit: return 5 + case .senseVoiceSmall: return 4 case .nemotronOffline: return 5 case .nemotronStreaming, .nemotronStreaming320: return 4 case .appleSpeech: return 4 @@ -4092,6 +4104,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 1.0 case .qwen3Asr: return 0.45 case .cohereTranscribeSixBit: return 0.85 + case .senseVoiceSmall: return 0.95 case .nemotronOffline: return 0.85 case .nemotronStreaming, .nemotronStreaming320: return 1.0 case .appleSpeech: return 0.60 @@ -4113,6 +4126,7 @@ final class SettingsStore: ObservableObject { case .parakeetRealtime: return 0.75 case .qwen3Asr: return 0.90 case .cohereTranscribeSixBit: return 0.98 + case .senseVoiceSmall: return 0.90 case .nemotronOffline: return 0.90 case .nemotronStreaming, .nemotronStreaming320: return 0.85 case .appleSpeech: return 0.60 @@ -4143,7 +4157,7 @@ final class SettingsStore: ObservableObject { /// Optimization level for Apple Silicon (for display) var appleSiliconOptimized: Bool { switch self { - case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeechAnalyzer: + case .parakeetTDT, .parakeetTDTv2, .parakeetRealtime, .qwen3Asr, .cohereTranscribeSixBit, .senseVoiceSmall, .nemotronOffline, .nemotronStreaming, .nemotronStreaming320, .appleSpeechAnalyzer: return true default: return false @@ -4154,8 +4168,8 @@ final class SettingsStore: ObservableObject { /// Large Whisper models are too slow for streaming, so they only do final transcription on stop. var supportsStreaming: Bool { switch self { - case .qwen3Asr, .whisperMedium, .whisperLargeTurbo, .whisperLarge: - return false // Too slow for real-time chunk processing + case .qwen3Asr, .senseVoiceSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: + return false // Non-streaming / too slow for real-time chunk processing default: return true // All other models support streaming } @@ -4198,6 +4212,7 @@ final class SettingsStore: ObservableObject { case openai = "OpenAI" case qwen = "Qwen" case cohere = "Cohere" + case funasr = "FunASR" } /// Which provider this model belongs to @@ -4211,6 +4226,8 @@ final class SettingsStore: ObservableObject { return .qwen case .cohereTranscribeSixBit: return .cohere + case .senseVoiceSmall: + return .funasr case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: return .openai } @@ -4261,6 +4278,13 @@ final class SettingsStore: ObservableObject { return false } return spec.validatesInstalledArtifacts(at: directory) + case .senseVoiceSmall: + #if arch(arm64) + guard let directory = SenseVoiceProvider.cacheDirectory() else { return false } + return SenseVoiceProvider.artifactsAreComplete(at: directory) + #else + return false + #endif case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: let hint: String switch self { @@ -4338,6 +4362,8 @@ final class SettingsStore: ObservableObject { return "Qwen" case .cohereTranscribeSixBit: return "Cohere" + case .senseVoiceSmall: + return "FunASR" case .appleSpeech, .appleSpeechAnalyzer: return "Apple" case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: @@ -4362,6 +4388,8 @@ final class SettingsStore: ObservableObject { return "#E67E22" case .cohereTranscribeSixBit: return "#FA6B3C" + case .senseVoiceSmall: + return "#FF6A00" // Alibaba Orange case .appleSpeech, .appleSpeechAnalyzer: return "#A2AAAD" // Apple Gray case .whisperTiny, .whisperBase, .whisperSmall, .whisperMedium, .whisperLargeTurbo, .whisperLarge: @@ -4557,6 +4585,7 @@ private extension SettingsStore { static let selectedSpeechModel = "SelectedSpeechModel" static let selectedCohereLanguage = "SelectedCohereLanguage" static let selectedNemotronLanguage = "SelectedNemotronLanguage" + static let senseVoiceLanguage = "SenseVoiceLanguage" static let selectedAppleSpeechLocaleIdentifier = "SelectedAppleSpeechLocaleIdentifier" static let externalCoreMLArtifactsDirectories = "ExternalCoreMLArtifactsDirectories" @@ -4696,6 +4725,8 @@ extension SettingsStore.SpeechModel { return "EN" case .cohereTranscribeSixBit: return "AR, DE, EL, EN, ES, FR, IT, JA, KO, NL, PL, PT, VI, ZH" + case .senseVoiceSmall: + return "YUE, ZH, EN, JA, KO + 50 more" case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return "40 language-locales" case .appleSpeechAnalyzer: @@ -4838,6 +4869,21 @@ extension SettingsStore { } } + var senseVoiceLanguage: SenseVoiceLanguage { + get { + if let rawValue = self.defaults.string(forKey: Keys.senseVoiceLanguage), + let language = SenseVoiceLanguage(rawValue: rawValue) + { + return language + } + return .cantonese + } + set { + objectWillChange.send() + self.defaults.set(newValue.rawValue, forKey: Keys.senseVoiceLanguage) + } + } + func externalCoreMLArtifactsDirectory(for model: SpeechModel) -> URL? { guard let spec = model.externalCoreMLSpec else { return nil } let paths = self.defaults.dictionary(forKey: Keys.externalCoreMLArtifactsDirectories) as? [String: String] ?? [:] diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index fbef8c16..60ad6eb2 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -149,6 +149,7 @@ final class ASRService: ObservableObject { private var parakeetRealtimeProvider: ParakeetRealtimeProvider? private var externalCoreMLProvider: ExternalCoreMLTranscriptionProvider? private var nemotronProviders: [NemotronProvider.Mode: NemotronProvider] = [:] + private var senseVoiceProvider: SenseVoiceProvider? private var whisperProvider: WhisperProvider? private var appleSpeechProvider: AppleSpeechProvider? /// Stored as Any? because @available cannot be applied to stored properties @@ -207,6 +208,8 @@ final class ASRService: ObservableObject { return self.getParakeetRealtimeProvider() case .cohereTranscribeSixBit: return self.getExternalCoreMLProvider() + case .senseVoiceSmall: + return self.getSenseVoiceProvider() case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return self.getNemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -259,6 +262,16 @@ final class ASRService: ObservableObject { return provider } + private func getSenseVoiceProvider() -> SenseVoiceProvider { + if let existing = self.senseVoiceProvider { + return existing + } + let provider = SenseVoiceProvider() + self.senseVoiceProvider = provider + DebugLogger.shared.info("ASRService: Created SenseVoice provider", source: "ASRService") + return provider + } + private func getWhisperProvider() -> WhisperProvider { if let existing = whisperProvider { return existing @@ -408,6 +421,8 @@ final class ASRService: ObservableObject { return ParakeetRealtimeProvider() case .cohereTranscribeSixBit: return ExternalCoreMLTranscriptionProvider(modelOverride: model) + case .senseVoiceSmall: + return SenseVoiceProvider(modelOverride: model) case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return NemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -525,6 +540,7 @@ final class ASRService: ObservableObject { self.fluidAudioProvider = nil self.parakeetRealtimeProvider = nil self.externalCoreMLProvider = nil + self.senseVoiceProvider = nil self.whisperProvider = nil self.appleSpeechProvider = nil self._appleSpeechAnalyzerProvider = nil diff --git a/Sources/Fluid/Services/SenseVoiceProvider.swift b/Sources/Fluid/Services/SenseVoiceProvider.swift new file mode 100644 index 00000000..325ef248 --- /dev/null +++ b/Sources/Fluid/Services/SenseVoiceProvider.swift @@ -0,0 +1,511 @@ +import Foundation + +#if arch(arm64) +@preconcurrency import CoreML + +/// TranscriptionProvider for SenseVoiceSmall (FunASR) — a non-autoregressive +/// multilingual ASR model with strong Cantonese accuracy, running on the Apple +/// Neural Engine via CoreML. +/// +/// The 3-stage CoreML pipeline + greedy-CTC decode is vendored from +/// FluidInference/FluidAudio (Apache-2.0), adapted to FluidVoice's own +/// `HuggingFaceModelDownloader` and settings because the pinned FluidAudio fork +/// does not include the SenseVoice module. Model artifacts come from +/// `FluidInference/sensevoice-small-coreml` (int8 encoder variant). +/// +/// See: https://github.com/FluidInference/FluidAudio (ASR/SenseVoice) +final class SenseVoiceProvider: TranscriptionProvider { + let name = "SenseVoice Small" + var isAvailable: Bool { true } + private(set) var isReady: Bool = false + + // MARK: - Repository / artifacts (int8 encoder) + + private let repositoryOwner = "FluidInference" + private let repositoryName = "sensevoice-small-coreml" + // Pinned to a specific commit for reproducibility: tracking `main` would let an + // upstream re-upload silently deliver a different (possibly incompatible) encoder to + // fresh installs. Bump this deliberately when adopting a new model revision. + private let repositoryRevision = "cdea3526163035c19915d4a10268992d018ebd46" + + static let preprocessorFile = "SenseVoicePreprocessor.mlmodelc" + static let encoderFile = "SenseVoiceSmall_int8.mlmodelc" + static let vocabularyFile = "vocab.json" + static let requiredItems: [HuggingFaceModelDownloader.ModelItem] = [ + .init(path: preprocessorFile, isDirectory: true), + .init(path: encoderFile, isDirectory: true), + .init(path: vocabularyFile, isDirectory: false), + ] + private static var requiredRelativePaths: [String] { + self.requiredItems.map { $0.path } + } + + private var engine: SenseVoiceEngine? + + // MARK: - Long-audio chunking (samples @ 16 kHz) + + /// Hard cap per pass. The CoreML preprocessor's waveform input is constrained to + /// 3200…480000 samples (0.2…30 s); anything longer is rejected outright, so this + /// sits safely below 480000 rather than at the encoder's ~108 s frame limit. + private static let maxChunkSamples = 460_000 + /// Never emit a chunk shorter than this (1 s), so a silence-seek can't collapse. + private static let minChunkSamples = 16_000 + /// How far back from the hard cap to hunt for a quiet cut point (2 s). + private static let boundarySearchRadiusSamples = 32_000 + private static let boundaryAnalysisWindowSamples = 1_280 + private static let boundaryAnalysisStrideSamples = 320 + + /// Optional model override (unused today — SenseVoice has a single variant — + /// but kept for parity with the other providers' download plumbing). + var modelOverride: SettingsStore.SpeechModel? + init(modelOverride: SettingsStore.SpeechModel? = nil) { + self.modelOverride = modelOverride + } + + static func cacheDirectory() -> URL? { + FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first? + .appendingPathComponent("sensevoice-small-coreml", isDirectory: true) + } + + static func artifactsAreComplete(at directory: URL) -> Bool { + HuggingFaceModelDownloader.artifactsAreComplete(root: directory, items: self.requiredItems) + } + + func modelsExistOnDisk() -> Bool { + guard let dir = Self.cacheDirectory() else { return false } + return Self.artifactsAreComplete(at: dir) + } + + // MARK: - Preparation + + func prepare(progressHandler: ((ModelPreparationProgress) -> Void)? = nil) async throws { + try Task.checkCancellation() + guard self.isReady == false else { return } + guard let dir = Self.cacheDirectory() else { + throw Self.makeError("Unable to resolve a cache directory for \(self.name).") + } + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + + // Re-sniff present artifacts: a proxy can cache an HTML block page (HTTP 200) in + // place of a model file, so a file-existence check alone would trust a corrupt + // cache forever. Force a re-download when that happens (see #353). + let modelsPresent = self.modelsExistOnDisk() + let cachedArtifactsCorrupt = modelsPresent + && HuggingFaceModelDownloader.cachedPayloadContainsMarkup( + root: dir, relativePaths: Self.requiredRelativePaths) + + if modelsPresent, !cachedArtifactsCorrupt { + DebugLogger.shared.info( + "SenseVoice: artifacts present at \(dir.path); skipping download", + source: "SenseVoice" + ) + } else { + if cachedArtifactsCorrupt { + DebugLogger.shared.warning( + "SenseVoice: cached artifacts at \(dir.path) contain an HTML/markup payload (corrupt); re-downloading", + source: "SenseVoice" + ) + } + DebugLogger.shared.info( + "SenseVoice: artifacts missing; downloading from \(self.repositoryOwner)/\(self.repositoryName)", + source: "SenseVoice" + ) + progressHandler?(.preparingDownload) + let downloader = HuggingFaceModelDownloader( + owner: self.repositoryOwner, + repo: self.repositoryName, + revision: self.repositoryRevision, + requiredItems: Self.requiredItems + ) + try await downloader.ensureModelsPresent(at: dir) { progress, _ in + progressHandler?(.downloading(progress)) + } + try Task.checkCancellation() + guard self.modelsExistOnDisk() else { + throw Self.makeError("SenseVoice artifacts incomplete after download at \(dir.path).") + } + } + + progressHandler?(.optimizing) + try Task.checkCancellation() + let engine = try SenseVoiceEngine( + directory: dir, + preprocessorName: Self.preprocessorFile, + encoderName: Self.encoderFile, + vocabularyFile: Self.vocabularyFile + ) + try Task.checkCancellation() + self.engine = engine + self.isReady = true + progressHandler?(.loading) + DebugLogger.shared.info("SenseVoice: provider ready", source: "SenseVoice") + } + + // MARK: - Transcription + + func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult { + guard let engine = self.engine else { + throw Self.makeError("SenseVoice provider is not ready.") + } + guard samples.isEmpty == false else { + return ASRTranscriptionResult(text: "", confidence: 0) + } + + let language = SettingsStore.shared.senseVoiceLanguage.embedIndex + let startedAt = Date().timeIntervalSince1970 + + let text: String + if samples.count <= Self.maxChunkSamples { + text = try await engine.transcribe(audio: samples, language: language) + } else { + // SenseVoice is non-streaming with a fixed ~30 s max input; split long audio + // at quiet points and stitch the pieces so nothing past the cap is dropped. + text = try await self.transcribeChunked(samples, language: language, engine: engine) + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + self.logBenchmark(samples: samples, text: trimmed, startedAt: startedAt) + // Return the trimmed text so downstream consumers get the same clean string used + // for the confidence check — matching the other providers' contract. + return ASRTranscriptionResult(text: trimmed, confidence: trimmed.isEmpty ? 0 : 1) + } + + /// Transcribe audio longer than one encoder pass by cutting it into + /// silence-aligned chunks (≤ `maxChunkSamples`) and joining the results. + private func transcribeChunked( + _ samples: [Float], language: Int32, engine: SenseVoiceEngine + ) async throws -> String { + var pieces: [String] = [] + var offset = 0 + while offset < samples.count { + try Task.checkCancellation() + let end = Self.chunkEnd(in: samples, offset: offset) + let chunk = Array(samples[offset.. String { + var result = "" + for piece in pieces where piece.isEmpty == false { + if let last = result.unicodeScalars.last, + let first = piece.unicodeScalars.first, + !isCJKScalar(last), !isCJKScalar(first) + { + result += " " + } + result += piece + } + return result + } + + /// Whether a scalar belongs to a script written without inter-word spaces + /// (CJK ideographs + Kana + Hangul), used to pick the chunk-join separator. + static func isCJKScalar(_ scalar: Unicode.Scalar) -> Bool { + switch scalar.value { + case 0x3040...0x30FF, // Hiragana, Katakana + 0x3400...0x4DBF, // CJK Extension A + 0x4E00...0x9FFF, // CJK Unified Ideographs + 0xAC00...0xD7A3, // Hangul syllables + 0xF900...0xFAFF, // CJK Compatibility Ideographs + 0x20000...0x2FA1F: // CJK Extensions B–F + Supplement + return true + default: + return false + } + } + + /// End index (exclusive) of the chunk starting at `offset`. For a non-final + /// chunk it seeks the quietest sample near the hard cap so cuts land in silence + /// rather than mid-syllable; the search never runs past the cap, so every chunk + /// stays within the encoder's limit. + private static func chunkEnd(in samples: [Float], offset: Int) -> Int { + let maxEnd = min(offset + self.maxChunkSamples, samples.count) + guard maxEnd < samples.count else { return samples.count } + + let lowerBound = max(offset + self.minChunkSamples, maxEnd - self.boundarySearchRadiusSamples) + guard lowerBound < maxEnd else { return maxEnd } + return self.quietestBoundary(in: samples, lowerBound: lowerBound, upperBound: maxEnd) ?? maxEnd + } + + /// The sample offset in `[lowerBound, upperBound]` with the lowest local energy, + /// nudged toward `upperBound` to prefer longer chunks on ties. + private static func quietestBoundary(in samples: [Float], lowerBound: Int, upperBound: Int) -> Int? { + let halfWindow = max(1, self.boundaryAnalysisWindowSamples / 2) + let stride = max(1, self.boundaryAnalysisStrideSamples) + var bestBoundary: Int? + var bestScore = Float.greatestFiniteMagnitude + var boundary = lowerBound + + while boundary <= upperBound { + let windowStart = max(0, boundary - halfWindow) + let windowEnd = min(samples.count, boundary + halfWindow) + var energy: Float = 0 + for index in windowStart.. 0 ? Double(elapsedMs) / Double(audioMs) : 0 + DebugLogger.shared.info( + """ + ASR_BENCH provider_final_done model=sensevoice samples=\(samples.count) audioMs=\(audioMs) \ + elapsedMs=\(elapsedMs) textChars=\(text.count) rtf=\(String(format: "%.3f", rtf)) + """, + source: "ASRBenchmark" + ) + } + + private static func makeError(_ description: String) -> NSError { + NSError(domain: "SenseVoiceProvider", code: -1, userInfo: [NSLocalizedDescriptionKey: description]) + } +} + +// MARK: - SenseVoice CoreML engine + +/// Actor that owns the loaded CoreML models and runs the SenseVoiceSmall pipeline +/// off the main thread. Pipeline (vendored from FluidInference/FluidAudio): +/// waveform → [Preprocessor fp32/CPU] → 560-d LFR features → pad to the smallest +/// enumerated bucket → [encoder+CTC int8/ANE] → greedy CTC decode (drop blank 0, +/// collapse repeats) → SentencePiece detokenize → strip `<|...|>` meta tags. +actor SenseVoiceEngine { + /// Configuration constants mirroring the `FluidInference/sensevoice-small-coreml` conversion. + private enum Config { + static let featureDim = 560 + static let buckets = [128, 256, 512, 1024, 1800] + static let numQueryTokens = 4 + static let blankId = 0 + static let defaultTextNorm: Int32 = 15 // woitn (no inverse text-norm) + static let waveformScale: Float = 32_768.0 + static let sentencePieceWordBoundary = "\u{2581}" // "▁" + /// Preprocessor waveform input floor (0.2 s); shorter audio is zero-padded up to it. + static let minWaveformSamples = 3_200 + static var maxFrames: Int { buckets.last ?? 1800 } + static func pickBucket(forFrames frames: Int) -> Int { + for b in buckets where b >= frames { return b } + return buckets.last ?? 1800 + } + } + + private let preprocessor: MLModel + private let encoder: MLModel + private let vocabulary: [Int: String] + + init(directory: URL, preprocessorName: String, encoderName: String, vocabularyFile: String) throws { + // Preprocessor must run fp32 on CPU (power-spectrum/log exceed fp16 range, + // and its big identity convs fail ANE compile). The int8 encoder runs on ANE. + let cpuConfig = MLModelConfiguration() + cpuConfig.computeUnits = .cpuOnly + let aneConfig = MLModelConfiguration() + aneConfig.computeUnits = .cpuAndNeuralEngine + + self.preprocessor = try MLModel( + contentsOf: directory.appendingPathComponent(preprocessorName), configuration: cpuConfig) + self.encoder = try MLModel( + contentsOf: directory.appendingPathComponent(encoderName), configuration: aneConfig) + self.vocabulary = try Self.loadVocabulary( + from: directory.appendingPathComponent(vocabularyFile)) + } + + /// Transcribe 16 kHz mono float samples (in [-1, 1]) for the given language embed index. + func transcribe(audio: [Float], language: Int32) throws -> String { + let features = try self.runPreprocessor(audio: audio) + let (logits, validFrames) = try self.runEncoder(features: features, language: language) + return self.decode(logits: logits, validFrames: validFrames) + } + + // MARK: - Pipeline + + private func runPreprocessor(audio: [Float]) throws -> MLMultiArray { + // The preprocessor rejects waveforms below its 3200-sample (0.2 s) floor, so a very + // short recording or a small trailing chunk is zero-padded (silence) up to it. + let n = max(audio.count, Config.minWaveformSamples) + let waveform = try MLMultiArray(shape: [1, n as NSNumber], dataType: .float32) + let wptr = waveform.dataPointer.assumingMemoryBound(to: Float32.self) + memset(wptr, 0, n * MemoryLayout.size) + let scale = Config.waveformScale + for i in 0.. (MLMultiArray, Int) { + let dim = Config.featureDim + var t = features.shape[1].intValue + // Clamp to the largest enumerated bucket (~108 s). Longer audio is truncated; + // callers needing full coverage should chunk before this point. + if t > Config.maxFrames { + t = Config.maxFrames + } + let bucket = Config.pickBucket(forFrames: t) + + // Zero-padded [1, bucket, 560] with the first T feature frames copied in. + let speech = try MLMultiArray(shape: [1, bucket as NSNumber, dim as NSNumber], dataType: .float32) + let sptr = speech.dataPointer.assumingMemoryBound(to: Float32.self) + memset(sptr, 0, bucket * dim * MemoryLayout.size) + let count = t * dim + if features.dataType == .float32 { + memcpy(sptr, features.dataPointer, count * MemoryLayout.size) + } else { + for i in 0..<|emo|><|event|><|itn|>` tags. + private func decode(logits: MLMultiArray, validFrames: Int) -> String { + let vocab = logits.shape[2].intValue + let frames = min(validFrames, logits.shape[1].intValue) + var ids: [Int] = [] + var prev = -1 + + func appendArgmax(frameBase: (Int) -> Float) { + var best = 0 + var bestVal = frameBase(0) + for v in 1.. bestVal { + bestVal = x + best = v + } + } + if best != Config.blankId && best != prev { ids.append(best) } + prev = best + } + + if logits.dataType == .float32 { + let p = logits.dataPointer.assumingMemoryBound(to: Float32.self) + for t in 0..", with: "", options: .regularExpression) + .trimmingCharacters(in: .whitespaces) + } + + // MARK: - Loading + + private static func loadVocabulary(from url: URL) throws -> [Int: String] { + let data = try Data(contentsOf: url) + // Canonical format: JSON array ["", "", ...]. + if let arr = try? JSONSerialization.jsonObject(with: data) as? [String] { + var v: [Int: String] = [:] + v.reserveCapacity(arr.count) + for (i, tok) in arr.enumerated() { v[i] = tok } + return v + } + // Fallback: {"0": "", ...} dictionary. + if let dict = try? JSONSerialization.jsonObject(with: data) as? [String: String] { + var v: [Int: String] = [:] + v.reserveCapacity(dict.count) + for (k, tok) in dict { if let i = Int(k) { v[i] = tok } } + return v + } + throw Self.makeError("Failed to parse \(url.lastPathComponent) (expected array or dict)") + } + + private static func makeError(_ description: String) -> NSError { + NSError(domain: "SenseVoiceEngine", code: -1, userInfo: [NSLocalizedDescriptionKey: description]) + } +} + +#else + +/// Intel stub — SenseVoice's int8 encoder targets the Apple Neural Engine. +final class SenseVoiceProvider: TranscriptionProvider { + let name = "SenseVoice Small (Apple Silicon ONLY)" + var isAvailable: Bool { false } + private(set) var isReady: Bool = false + + init(modelOverride: SettingsStore.SpeechModel? = nil) {} + + func prepare(progressHandler: ((ModelPreparationProgress) -> Void)?) async throws { + throw NSError( + domain: "SenseVoiceProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "SenseVoice requires Apple Silicon."] + ) + } + + func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult { + throw NSError( + domain: "SenseVoiceProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "SenseVoice requires Apple Silicon."] + ) + } + + func modelsExistOnDisk() -> Bool { false } +} + +#endif diff --git a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift index 269c633e..3493d01f 100644 --- a/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift +++ b/Sources/Fluid/UI/AISettingsView+SpeechRecognition.swift @@ -569,6 +569,25 @@ extension VoiceEngineSettingsView { .buttonStyle(.plain) } else if model == .nemotronOffline || model == .nemotronStreaming || model == .nemotronStreaming320 { self.nemotronLanguagePickerButton + } else if model == .senseVoiceSmall { + Menu { + ForEach(SettingsStore.SenseVoiceLanguage.allCases) { language in + Button { + guard language != self.settings.senseVoiceLanguage else { return } + self.settings.senseVoiceLanguage = language + } label: { + HStack { + Text(language.displayName) + if language == self.settings.senseVoiceLanguage { + Image(systemName: "checkmark") + } + } + } + } + } label: { + self.languageChipLabel(self.settings.senseVoiceLanguage.displayName) + } + .buttonStyle(.plain) } }