diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index ea8d2f2c0..18488300b 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -352,6 +352,7 @@ class CloudSyncRepository @Inject constructor( // Global AI subtitle keys (non-profile-scoped, device-wide) private val subtitleAiEnabledKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_enabled") private val subtitleAiAutoSelectKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_auto_select") + private val subtitleAiFindBestMatchKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_ai_find_best_match") private val subtitleAiApiKeyKey = androidx.datastore.preferences.core.stringPreferencesKey("subtitle_ai_api_key") private val subtitleAiModelKey = androidx.datastore.preferences.core.stringPreferencesKey("subtitle_ai_model") private val subtitleRemoveHearingImpairedKey = androidx.datastore.preferences.core.booleanPreferencesKey("subtitle_remove_hearing_impaired") @@ -489,6 +490,7 @@ class CloudSyncRepository @Inject constructor( // Global AI subtitle settings (non-profile-scoped) root.put("subtitleAiEnabled", prefs[subtitleAiEnabledKey] ?: false) root.put("subtitleAiAutoSelect", prefs[subtitleAiAutoSelectKey] ?: false) + root.put("subtitleAiFindBestMatch", prefs[subtitleAiFindBestMatchKey] ?: false) root.put("subtitleAiApiKey", prefs[subtitleAiApiKeyKey] ?: "") root.put("subtitleAiModel", prefs[subtitleAiModelKey] ?: "GROQ_LLAMA_70B") root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) @@ -1227,6 +1229,7 @@ class CloudSyncRepository @Inject constructor( context.settingsDataStore.edit { prefs -> prefs[subtitleAiEnabledKey] = root.optBoolean("subtitleAiEnabled", false) prefs[subtitleAiAutoSelectKey] = root.optBoolean("subtitleAiAutoSelect", false) + prefs[subtitleAiFindBestMatchKey] = root.optBoolean("subtitleAiFindBestMatch", false) val apiKey = root.optString("subtitleAiApiKey", "") if (apiKey.isNotBlank()) prefs[subtitleAiApiKeyKey] = apiKey val model = root.optString("subtitleAiModel", "GROQ_LLAMA_70B") diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt index 9c28b10b3..f97888b2e 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/StreamRepository.kt @@ -40,6 +40,7 @@ import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.callbackFlow import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first @@ -1410,7 +1411,10 @@ class StreamRepository @Inject constructor( private val ADDON_EXTENDED_EPISODE_TIMEOUT_MS = 45_000L private val ADDON_EXTENDED_SINGLE_STREAM_REQUEST_TIMEOUT_MS = 35_000L // Subtitles should not block playback but need enough time on slow connections. - private val SUBTITLE_TIMEOUT_MS = 6_000L + // Generous because aggregator addons (AIOStreams) fan out to upstreams server-side and can + // take 10s+ cold (or 502 outright, then succeed warm — hence the retry). Fetches run in + // parallel and in the background, so a slow addon delays nothing. + private val SUBTITLE_TIMEOUT_MS = 20_000L // If addons return nothing, allow Xtream VOD lookup to recover playback. private val VOD_LOOKUP_TIMEOUT_MS = 6_000L // If addons already returned streams, keep VOD lookup shorter to avoid UI delay. @@ -2754,21 +2758,24 @@ class StreamRepository @Inject constructor( else -> "" } - subtitleAddons.flatMap { addon -> - runCatching { - withTimeout(SUBTITLE_TIMEOUT_MS) { - val addonUrl = addon.url ?: return@withTimeout emptyList() - val (baseUrl, queryParams) = getAddonBaseUrl(addonUrl) - val url = buildSubtitlesUrl( - baseUrl = baseUrl, - type = type, - id = contentId, - addonQueryParams = queryParams, - videoHash = videoHash, - videoSize = videoSize - ) - val response = streamApi.getSubtitles(url) - response.subtitles?.mapIndexed { index, sub -> + // Query addons in parallel — with a generous timeout, sequential fetches would stack + // each slow addon's latency onto the total. + subtitleAddons.map { addon -> + async { + suspend fun attempt(): List = runCatching { + withTimeout(SUBTITLE_TIMEOUT_MS) { + val addonUrl = addon.url ?: return@withTimeout emptyList() + val (baseUrl, queryParams) = getAddonBaseUrl(addonUrl) + val url = buildSubtitlesUrl( + baseUrl = baseUrl, + type = type, + id = contentId, + addonQueryParams = queryParams, + videoHash = videoHash, + videoSize = videoSize + ) + val response = streamApi.getSubtitles(url) + response.subtitles?.mapIndexed { index, sub -> val normalizedLang = normalizeLanguageCode(sub.lang) val langFallback = sub.lang?.trim()?.lowercase()?.take(2) ?: "und" Subtitle( @@ -2779,9 +2786,21 @@ class StreamRepository @Inject constructor( provider = addon.name ) } ?: emptyList() + } + }.onFailure { + Log.w("StreamRepository", "subtitles fetch failed addon=${addon.name} err=${it.message}") + }.getOrDefault(emptyList()) + + // Aggregator endpoints (AIOStreams) often hang or 502 on the first (cold) hit and + // succeed once their upstream fan-out is warm — one retry recovers those. + var subs = attempt() + if (subs.isEmpty()) { + delay(1_500) + subs = attempt() } - }.getOrDefault(emptyList()) - } + subs + } + }.awaitAll().flatten() } private fun buildSubtitlesUrl( diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt index ab12d5a75..07a45b632 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AiSubtitleRenderersFactory.kt @@ -6,6 +6,8 @@ import androidx.media3.common.text.Cue import androidx.media3.common.text.CueGroup import androidx.media3.exoplayer.DefaultRenderersFactory import androidx.media3.exoplayer.Renderer +import androidx.media3.exoplayer.audio.AudioSink +import androidx.media3.exoplayer.audio.DefaultAudioSink import androidx.media3.exoplayer.text.TextOutput import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -26,6 +28,52 @@ class AiSubtitleRenderersFactory( val syncOffsetUs = java.util.concurrent.atomic.AtomicLong(0L) + var audioCaptureProcessor: AudioCaptureProcessor? = null + private set + + private val offsetRenderers = mutableListOf() + + /** + * Cue-visible intervals (startMs to endMs) of the currently selected text track, read from + * ExoPlayer's buffered cue list via reflection — i.e. upcoming cues that haven't rendered yet. + * Lets "Find best match" build its built-in reference from prefetched cues instead of waiting + * for playback to reach them. Returns empty if nothing is buffered yet. + */ + fun extractBufferedReferenceIntervals(maxCount: Int): List> { + for (renderer in offsetRenderers) { + val intervals = renderer.extractBufferedIntervals(maxCount) + if (intervals.isNotEmpty()) return intervals + } + return emptyList() + } + + /** + * Text of the currently-buffered cues of the selected text track. Lets "Find best match" + * verify WHICH track the buffer belongs to (by comparing against the displayed candidate's + * own lines) — timing-based self-detection false-positived on subs cut from the same master. + */ + fun extractBufferedCueTexts(maxCount: Int): List { + for (renderer in offsetRenderers) { + val texts = renderer.extractAllCueTexts() + if (texts.isNotEmpty()) return texts.take(maxCount) + } + return emptyList() + } + + override fun buildAudioSink( + context: Context, + enableFloatOutput: Boolean, + enableAudioTrackPlaybackParams: Boolean + ): AudioSink { + val capture = AudioCaptureProcessor() + audioCaptureProcessor = capture + return DefaultAudioSink.Builder(context) + .setEnableFloatOutput(enableFloatOutput) + .setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams) + .setAudioProcessors(arrayOf(capture)) + .build() + } + override fun buildTextRenderers( context: Context, output: TextOutput, @@ -41,7 +89,7 @@ class AiSubtitleRenderersFactory( ) val startIndex = out.size super.buildTextRenderers(context, translatingOutput, outputLooper, extensionRendererMode, out) - val offsetRenderers = mutableListOf() + offsetRenderers.clear() for (index in startIndex until out.size) { val offsetRenderer = SubtitleOffsetRenderer( baseRenderer = out[index], @@ -287,6 +335,10 @@ private class SubtitleOffsetRenderer( * while TextRenderer.render() has the subtitle buffer populated. */ fun triggerPreTranslation() { + // No AI translation active → no lookahead. tryPeriodicLookahead already checks this; + // this first-cue trigger fires for any newly selected track (e.g. the reference track + // "find best match" selects with AI off) and must not spend API requests either. + if (!translationManager.isEnabled) return if (preTranslatedUpToUs != Long.MIN_VALUE && preTranslatedUpToUs > currentPositionUs + PREFETCH_TRIGGER_US) { return @@ -311,7 +363,7 @@ private class SubtitleOffsetRenderer( // ── Reflection-based cue extraction ────────────────────────────────────── - private fun extractAllCueTexts(): List { + fun extractAllCueTexts(): List { val removeHI = translationManager.removeHearingImpaired val texts = mutableSetOf() @@ -374,6 +426,104 @@ private class SubtitleOffsetRenderer( return texts.toList() } + /** Buffered cue intervals (startMs, endMs) for text-carrying cues, from the modern resolver. */ + fun extractBufferedIntervals(maxCount: Int): List> { + val intervals = ArrayList>() + // Cue buffer timestamps are on ExoPlayer's internal stream timeline, which adds a large + // base offset; subtract the renderer's stream offset to get 0-based media time. + val offsetMs = readStreamOffsetUs() / 1000L + try { + val resolverField = findField(baseRenderer.javaClass, "cuesResolver") + val resolver = resolverField?.get(baseRenderer) + if (resolver != null) { + // Media3 has multiple resolver impls chosen per track cue-replacement behavior + // (MergingCuesResolver, ReplacingCuesResolver) — probe the known field names. + for (candidate in listOf( + "cuesWithTimingList", "cuesWithTimings", + "cueGroupsByStartTime", "cueGroups", "cueGroupList", "groups" + )) { + val f = findField(resolver.javaClass, candidate) ?: continue + val v = f.get(resolver) ?: continue + val items: Collection<*> = when (v) { + is Map<*, *> -> v.values + is Collection<*> -> v + else -> continue + } + for (item in items) { + val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue + intervals.add((s - offsetMs) to (e - offsetMs)) + } + if (intervals.isNotEmpty()) break + } + if (intervals.isEmpty()) { + // Unknown impl/field name (new media3 version?) — scan every field for a + // cue-carrying collection, like the text extractor already does. + var cls: Class<*>? = resolver.javaClass + outer@ while (cls != null && cls != Any::class.java) { + for (f in cls.declaredFields) { + try { + f.isAccessible = true + val v = f.get(resolver) ?: continue + val items: Collection<*> = when (v) { + is Map<*, *> -> v.values + is Collection<*> -> v + else -> continue + } + for (item in items) { + val (s, e) = item?.let(::intervalFromCueWrapper) ?: continue + intervals.add((s - offsetMs) to (e - offsetMs)) + } + if (intervals.isNotEmpty()) break@outer + } catch (_: Exception) { + } + } + cls = cls.superclass + } + } + } + } catch (_: Exception) { + } + return intervals.filter { it.first >= 0 }.distinct().sortedBy { it.first }.take(maxCount) + } + + /** The renderer's stream offset (µs) — the base added to buffer sample timestamps. */ + private fun readStreamOffsetUs(): Long { + for (name in listOf("streamOffsetUs", "outputStreamOffsetUs")) { + val v = runCatching { findField(baseRenderer.javaClass, name)?.getLong(baseRenderer) }.getOrNull() + if (v != null && v > 0L) return v + } + return 0L + } + + private fun intervalFromCueWrapper(obj: Any): Pair? { + val cues: List? = when (obj) { + is CueGroup -> obj.cues + is List<*> -> obj.filterIsInstance() + else -> (findField(obj.javaClass, "cues")?.get(obj) as? List<*>)?.filterIsInstance() + } + if (cues.isNullOrEmpty() || cues.none { !it.text?.toString()?.trim().isNullOrBlank() }) return null + + val startUs: Long + val endUs: Long + if (obj is CueGroup) { + startUs = obj.presentationTimeUs + endUs = startUs + 2_000_000L // CueGroup carries no duration — assume 2s on screen + } else { + val start = runCatching { findField(obj.javaClass, "startTimeUs")?.getLong(obj) }.getOrNull() ?: return null + // REPLACE-behavior tracks (ReplacingCuesResolver, e.g. MKV SubRip) carry + // durationUs = C.TIME_UNSET (large negative) — each cue lasts until replaced. The + // negative value made endUs < startUs and every item got rejected, silently killing + // the buffered fast path for that whole content category. Fall back to a nominal + // duration: the start timestamp carries the alignment signal. + val dur = runCatching { findField(obj.javaClass, "durationUs")?.getLong(obj) }.getOrNull() + ?.takeIf { it > 0L } ?: 2_000_000L + startUs = start + endUs = start + dur + } + if (endUs <= startUs) return null + return (startUs / 1000L) to (endUs / 1000L) + } + private fun extractFromCollectionOrMap(v: Any, texts: MutableSet, removeHI: Boolean): Int { var count = 0 when (v) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt new file mode 100644 index 000000000..e37bd5384 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/AudioCaptureProcessor.kt @@ -0,0 +1,141 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Log +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.audio.BaseAudioProcessor +import androidx.media3.common.audio.AudioProcessor.AudioFormat +import androidx.media3.common.audio.AudioProcessor.UnhandledAudioFormatException +import androidx.media3.common.util.UnstableApi +import java.nio.ByteBuffer +import java.nio.ByteOrder + +private const val TARGET_RATE = 16000 +private const val CHUNK_SAMPLES = TARGET_RATE / 10 // 100 ms per chunk + +@OptIn(UnstableApi::class) +class AudioCaptureProcessor : BaseAudioProcessor() { + + /** Set to receive (pcm16kHz, captureTimeMs) pairs; null = capture inactive. */ + @Volatile var onChunk: ((ByteArray, Long) -> Unit)? = null + + // Resampler state + private var srcStep = 1.0 // input samples per output sample + private var srcFrac = 0.0 // fractional position within current input block + private val accum = ArrayList(CHUNK_SAMPLES * 2) + private var prevMono: Short = 0 // last sample of previous block for interpolation + + @Throws(UnhandledAudioFormatException::class) + override fun onConfigure(inputAudioFormat: AudioFormat): AudioFormat { + Log.d("AudioCapture", "onConfigure encoding=${inputAudioFormat.encoding} " + + "rate=${inputAudioFormat.sampleRate} ch=${inputAudioFormat.channelCount} " + + "onChunk=${onChunk != null}") + if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT && + inputAudioFormat.encoding != C.ENCODING_PCM_FLOAT) { + // Still passthrough — return same format, just won't capture. + } + srcStep = if (inputAudioFormat.sampleRate > 0) + inputAudioFormat.sampleRate.toDouble() / TARGET_RATE else 1.0 + srcFrac = 0.0 + accum.clear() + prevMono = 0 + return inputAudioFormat + } + + override fun onFlush() { + srcFrac = 0.0 + accum.clear() + prevMono = 0 + } + + override fun onReset() { + srcFrac = 0.0 + accum.clear() + prevMono = 0 + } + + private var queueInputCallCount = 0 + override fun queueInput(inputBuffer: ByteBuffer) { + queueInputCallCount++ + if (queueInputCallCount <= 3 || queueInputCallCount % 500 == 0) { + Log.d("AudioCapture", "queueInput #$queueInputCallCount remaining=${inputBuffer.remaining()} " + + "onChunk=${onChunk != null} encoding=${inputAudioFormat.encoding}") + } + val size = inputBuffer.remaining() + val bytes = ByteArray(size) + inputBuffer.get(bytes) // advances position to limit + + // Write passthrough output + replaceOutputBuffer(size).put(bytes).flip() + + val callback = onChunk ?: return + val fmt = inputAudioFormat + if (fmt == AudioFormat.NOT_SET) return + + when (fmt.encoding) { + C.ENCODING_PCM_16BIT -> processPcm16(bytes, fmt.channelCount, callback) + C.ENCODING_PCM_FLOAT -> processPcmFloat(bytes, fmt.channelCount, callback) + // Other encodings: skip capture, passthrough only + } + } + + private fun processPcm16(bytes: ByteArray, channels: Int, callback: (ByteArray, Long) -> Unit) { + val shorts = ShortArray(bytes.size / 2) { i -> + ByteBuffer.wrap(bytes, i * 2, 2).order(ByteOrder.LITTLE_ENDIAN).short + } + resampleAndEmit(mixToMono(shorts, channels), callback) + } + + private fun processPcmFloat(bytes: ByteArray, channels: Int, callback: (ByteArray, Long) -> Unit) { + val floatBuf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer() + val shorts = ShortArray(floatBuf.remaining()) { floatBuf.get().coerceIn(-1f, 1f).let { (it * Short.MAX_VALUE).toInt().toShort() } } + resampleAndEmit(mixToMono(shorts, channels), callback) + } + + private fun mixToMono(samples: ShortArray, channels: Int): ShortArray { + if (channels == 1) return samples + if (channels == 6) { + // 5.1 channel order: FL(0), FR(1), C(2), LFE(3), BL(4), BR(5) + // ITU-R BS.775: mono = C*1.0 + (FL+FR)*0.707 + (BL+BR)*0.707 + LFE*0.0 + // Integer arithmetic: multiply by 1000, sum of weights ≈ 3828, divide by 3828 + return ShortArray(samples.size / 6) { i -> + val fl = samples[i * 6 + 0].toLong() + val fr = samples[i * 6 + 1].toLong() + val c = samples[i * 6 + 2].toLong() + val bl = samples[i * 6 + 4].toLong() + val br = samples[i * 6 + 5].toLong() + ((c * 1000 + (fl + fr + bl + br) * 707) / 3828).toInt().toShort() + } + } + return ShortArray(samples.size / channels) { i -> + var sum = 0L + for (c in 0 until channels) sum += samples[i * channels + c] + (sum / channels).toShort() + } + } + + private fun resampleAndEmit(mono: ShortArray, callback: (ByteArray, Long) -> Unit) { + if (mono.isEmpty()) return + var pos = srcFrac + while (pos < mono.size) { + val idx = pos.toInt().coerceIn(0, mono.size - 1) + val next = if (idx + 1 < mono.size) mono[idx + 1] else if (mono.size > 0) prevMono else mono[0] + val frac = pos - idx + val sample = ((mono[idx] * (1.0 - frac)) + (next * frac)).toInt().toShort() + accum.add(sample) + if (accum.size >= CHUNK_SAMPLES) { + callback(shortsToBytes(accum), System.currentTimeMillis()) + accum.clear() + } + pos += srcStep + } + srcFrac = pos - mono.size + if (mono.isNotEmpty()) prevMono = mono.last() + } + + private fun shortsToBytes(shorts: List): ByteArray { + val buf = ByteBuffer.allocate(shorts.size * 2).order(ByteOrder.LITTLE_ENDIAN) + for (s in shorts) buf.putShort(s) + return buf.array() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt new file mode 100644 index 000000000..acd29ac87 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/GeminiLiveTranslationService.kt @@ -0,0 +1,319 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Base64 +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import org.json.JSONArray +import org.json.JSONObject +import java.util.concurrent.TimeUnit + +private const val TAG = "GeminiLive" +// Specialized streaming speech-to-text-translation model: lowest latency (~0.8s) because it +// translates continuously without waiting for speech turns. Trade-off: it ignores system +// instructions and exposes no quality/style/gender controls, so translation quality is fixed. +// The general (prompt-steerable) Live models were evaluated but rejected: the TEXT-capable +// ones aren't available on this key, and the reachable native-audio model adds ~3s latency. +private const val MODEL = "models/gemini-3.5-live-translate-preview" +private const val WS_BASE = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" +private const val MIME_PCM_16K = "audio/pcm;rate=16000" +private const val CLEAR_DELAY_MS = 3_000L +// Minimum time a completed sentence stays on screen before the next queued one replaces it. +private const val MIN_LINE_MS = 900L +// If we fall this far behind on rapid dialogue, drop the oldest lines to catch back up. +private const val MAX_PENDING_LINES = 3 +private val SENTENCE_TERMINATORS = charArrayOf('.', '!', '?', '…') +// Split a run of completed text into individual sentences (break after each terminator). +private val SENTENCE_SPLIT = Regex("(?<=[.!?…])\\s+") + +enum class GeminiLiveState { DISCONNECTED, CONNECTING, READY, ERROR } + +class GeminiLiveTranslationService( + private val apiKeyProvider: () -> String, + private val scope: CoroutineScope +) { + /** BCP-47 code the live translation outputs (the user's preferred subtitle language). */ + @Volatile var targetLanguageCode: String = "he" + + private val _text = MutableStateFlow(null) + val translatedText: StateFlow = _text.asStateFlow() + + private val _state = MutableStateFlow(GeminiLiveState.DISCONNECTED) + val state: StateFlow = _state.asStateFlow() + + private val _errorMessage = MutableStateFlow(null) + val errorMessage: StateFlow = _errorMessage.asStateFlow() + + private var ws: WebSocket? = null + private val audioQueue = Channel(Channel.UNLIMITED) + private var senderJob: Job? = null + private var clearJob: Job? = null + + private var sentenceStartMs = 0L // wall time when first chunk was queued after last sentence end + private var firstFragmentLogged = false + private var lastFragmentTimeMs = 0L + + // Segmentation: show one sentence at a time so lines from different speakers/utterances + // never merge onto a single subtitle line. Completed sentences queue up and are drained + // one at a time; the trailing in-progress text shows live once the queue is empty. + private val pendingLines = ArrayDeque() + private var livePartial = StringBuilder() + private var lineJob: Job? = null + + private val client = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(0, TimeUnit.SECONDS) + .build() + + fun connect() { + Log.d(TAG, "connect() state=${_state.value}") + if (_state.value == GeminiLiveState.CONNECTING || _state.value == GeminiLiveState.READY) return + val apiKey = apiKeyProvider() + if (apiKey.isBlank()) { + _state.value = GeminiLiveState.ERROR + _errorMessage.value = "API key missing" + return + } + _state.value = GeminiLiveState.CONNECTING + _errorMessage.value = null + + val request = Request.Builder().url("$WS_BASE?key=$apiKey").build() + ws = client.newWebSocket(request, object : WebSocketListener() { + override fun onOpen(socket: WebSocket, response: Response) { + Log.d(TAG, "WS open, sending setup") + socket.send(buildSetupMessage()) + _state.value = GeminiLiveState.READY + startSender() + } + override fun onMessage(socket: WebSocket, text: String) { + handleMessage(text) + } + override fun onMessage(socket: WebSocket, bytes: ByteString) { + handleMessage(bytes.utf8()) + } + override fun onFailure(socket: WebSocket, t: Throwable, response: Response?) { + val body = try { response?.body?.string() } catch (_: Exception) { null } + Log.e(TAG, "WS failure code=${response?.code} body=$body err=${t.message}") + _state.value = GeminiLiveState.ERROR + _errorMessage.value = t.message ?: "Connection failed" + stopSender() + } + override fun onClosed(socket: WebSocket, code: Int, reason: String) { + Log.e(TAG, "WS closed code=$code reason=$reason") + _state.value = GeminiLiveState.DISCONNECTED + stopSender() + } + }) + } + + private fun buildSetupMessage() = JSONObject().apply { + put("setup", JSONObject().apply { + put("model", MODEL) + put("generation_config", JSONObject().apply { + // TEXT output keeps latency low (no dubbed-speech generation). + put("response_modalities", JSONArray().apply { put("TEXT") }) + // The translate model's only real control: target language. It ignores + // system instructions and has no quality/style/gender knobs, so we don't send any. + put("translation_config", JSONObject().apply { + put("target_language_code", targetLanguageCode) + }) + }) + }) + }.toString() + + /** Concatenate all text parts from serverContent.modelTurn (TEXT response modality). */ + private fun extractModelTurnText(content: JSONObject): String? { + val modelTurn = content.optJSONObject("modelTurn") + ?: content.optJSONObject("model_turn") ?: return null + val parts = modelTurn.optJSONArray("parts") ?: return null + val sb = StringBuilder() + for (i in 0 until parts.length()) { + val text = parts.optJSONObject(i)?.optString("text", "").orEmpty() + if (text.isNotEmpty()) sb.append(text) + } + return sb.toString().ifBlank { null } + } + + private fun handleMessage(raw: String) { + try { + val json = JSONObject(raw) + if (json.has("setupComplete") || json.has("setup_complete")) { + Log.d(TAG, "setupComplete") + return + } + val content = json.optJSONObject("serverContent") + ?: json.optJSONObject("server_content") ?: return + + // With TEXT modality the translated text arrives in modelTurn.parts[].text. + // Fall back to outputTranscription for the AUDIO-modality shape so either works. + val fragment = extractModelTurnText(content) + ?: content.optJSONObject("outputTranscription")?.optString("text", "") + ?: content.optJSONObject("output_transcription")?.optString("text", "") + + if (!fragment.isNullOrBlank()) { + val now = System.currentTimeMillis() + // A gap of >3s means the previous utterance ended without a clean boundary — reset + // so stale text doesn't bleed into the next speaker's line. + if (lastFragmentTimeMs > 0 && now - lastFragmentTimeMs > 3_000L) { + resetSegmentation() + } + lastFragmentTimeMs = now + if (!firstFragmentLogged) { + val elapsedMs = if (sentenceStartMs > 0L) now - sentenceStartMs else -1L + Log.i(TAG, "⏱ first fragment after ${elapsedMs}ms | text=\"${fragment.trim()}\"") + firstFragmentLogged = true + } + + // TEXT parts are incremental token chunks that carry their own spacing, + // so append verbatim (no trim / no injected space) to avoid splitting words. + livePartial.append(fragment) + extractCompletedSentences() + Log.d(TAG, "live: partial=\"${livePartial.toString().trim()}\" queued=${pendingLines.size}") + startDrainer() + } + + // Turn-based (general) model: a completed turn is a clean utterance boundary — flush + // the trailing text as its own line and reset per-turn latency timing. + if (content.optBoolean("turnComplete", false) || content.optBoolean("turn_complete", false)) { + onTurnComplete() + } + } catch (e: Exception) { + Log.w(TAG, "parse error: ${e.message}") + } + } + + private fun onTurnComplete() { + val rest = livePartial.toString().trim() + if (rest.isNotEmpty()) { + pendingLines.addLast(rest) + while (pendingLines.size > MAX_PENDING_LINES) pendingLines.removeFirst() + livePartial = StringBuilder() + startDrainer() + } + sentenceStartMs = 0L + firstFragmentLogged = false + } + + /** + * Move every complete sentence out of [livePartial] into the [pendingLines] queue, leaving + * only the trailing in-progress fragment. This is what stops two speakers' sentences from + * ever being concatenated onto one line — each terminated sentence becomes its own line. + */ + private fun extractCompletedSentences() { + val s = livePartial.toString() + val lastTerm = s.indexOfLast { it in SENTENCE_TERMINATORS } + if (lastTerm < 0) return + val completed = s.substring(0, lastTerm + 1) + val rest = s.substring(lastTerm + 1) + for (sentence in SENTENCE_SPLIT.split(completed)) { + val t = sentence.trim() + if (t.isNotEmpty()) pendingLines.addLast(t) + } + livePartial = StringBuilder(rest) + // Bound the backlog so rapid dialogue doesn't drift ever further behind. + while (pendingLines.size > MAX_PENDING_LINES) pendingLines.removeFirst() + } + + /** + * Drains queued sentences one at a time, holding each for [MIN_LINE_MS] so it's readable, + * then shows the live in-progress text once caught up. Only one drainer runs at a time; + * new sentences appended while it's draining are picked up by the loop. + */ + private fun startDrainer() { + if (lineJob?.isActive == true) return + lineJob = scope.launch { + while (pendingLines.isNotEmpty()) { + val line = pendingLines.removeFirst() + _text.value = line + sentenceStartMs = 0L + firstFragmentLogged = false + delay(MIN_LINE_MS) + } + // Caught up — show whatever partial sentence is currently being spoken. + val partial = livePartial.toString().trim() + if (partial.isNotEmpty()) _text.value = partial + scheduleClear() + } + } + + private fun scheduleClear() { + clearJob?.cancel() + clearJob = scope.launch { + delay(CLEAR_DELAY_MS) + if (pendingLines.isEmpty() && livePartial.isBlank()) _text.value = null + } + } + + private fun resetSegmentation() { + lineJob?.cancel() + clearJob?.cancel() + pendingLines.clear() + livePartial = StringBuilder() + sentenceStartMs = 0L + firstFragmentLogged = false + _text.value = null + } + + fun sendAudioChunk(pcm16Bytes: ByteArray, captureTimeMs: Long) { + if (_state.value != GeminiLiveState.READY) return + if (sentenceStartMs == 0L) { + sentenceStartMs = System.currentTimeMillis() + firstFragmentLogged = false + } + audioQueue.trySend(pcm16Bytes) + } + + private fun startSender() { + Log.d(TAG, "startSender launched") + var n = 0 + senderJob = scope.launch { + for (chunk in audioQueue) { + val socket = ws ?: break + n++ + if (n <= 3 || n % 50 == 0) Log.d(TAG, "chunk #$n") + val encoded = Base64.encodeToString(chunk, Base64.NO_WRAP) + socket.send(JSONObject().apply { + put("realtimeInput", JSONObject().apply { + put("audio", JSONObject().apply { + put("data", encoded) + put("mimeType", MIME_PCM_16K) + }) + }) + }.toString()) + } + Log.d(TAG, "sender ended after $n chunks") + } + } + + private fun stopSender() { + senderJob?.cancel() + senderJob = null + } + + fun disconnect() { + stopSender() + lineJob?.cancel() + clearJob?.cancel() + ws?.close(1000, "stopped") + ws = null + _state.value = GeminiLiveState.DISCONNECTED + _text.value = null + pendingLines.clear() + livePartial = StringBuilder() + sentenceStartMs = 0L + firstFragmentLogged = false + lastFragmentTimeMs = 0L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt index 08aab28b9..2c674d3c8 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerScreen.kt @@ -71,6 +71,8 @@ import androidx.compose.material.icons.filled.Settings import androidx.compose.material.icons.filled.VolumeDown import androidx.compose.material.icons.filled.VolumeMute import androidx.compose.material.icons.filled.VolumeUp +import androidx.compose.material.icons.filled.Mic +import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.unit.Dp @@ -445,12 +447,17 @@ fun PlayerScreen( )) } .toMutableList() - // When AI is available but no subtitles exist in the target language yet, inject a - // synthetic empty group so the AI option is reachable in the picker. - if (uiState.isAiAvailable && uiState.aiTargetLanguageName.isNotBlank() && - groups.none { (name, _) -> name.equals(uiState.aiTargetLanguageName, ignoreCase = true) }) { - groups.add(0, Pair(uiState.aiTargetLanguageName, emptyList())) + // When no subtitles exist in the target language yet, inject a synthetic empty group so + // the "Find Best Match" entry (AI-independent) and the AI option stay reachable. + val headerGroupName = uiState.matchLanguageName.ifBlank { + if (uiState.isAiAvailable) uiState.aiTargetLanguageName else "" } + if (headerGroupName.isNotBlank() && + groups.none { (name, _) -> name.equals(headerGroupName, ignoreCase = true) }) { + groups.add(0, Pair(headerGroupName, emptyList())) + } + // "Find Best Match" is rendered inside the target-language (e.g. Hebrew) group as its + // first item, alongside the AI translation option — not as a global picker entry. groups.toList() } // Audio tracks from ExoPlayer @@ -757,10 +764,16 @@ fun PlayerScreen( .setDefaultRequestProperties(baseRequestHeaders) } val mediaCache = remember(context) { PlaybackCacheSingleton.getInstance(context) } - val cacheDataSourceFactory = remember(httpDataSourceFactory, mediaCache) { + // Wrap the OkHttp factory so file:// URIs also work — matched subtitles are served from a + // local cache file (already downloaded by the scan) instead of re-fetching addon servers. + // http/https still routes through the same OkHttp client as before. + val fileCapableDataSourceFactory = remember(httpDataSourceFactory) { + androidx.media3.datasource.DefaultDataSource.Factory(context, httpDataSourceFactory) + } + val cacheDataSourceFactory = remember(fileCapableDataSourceFactory, mediaCache) { CacheDataSource.Factory() .setCache(mediaCache) - .setUpstreamDataSourceFactory(httpDataSourceFactory) + .setUpstreamDataSourceFactory(fileCapableDataSourceFactory) .setFlags(CacheDataSource.FLAG_IGNORE_CACHE_ON_ERROR) } // Non-cached factory for heavy/debrid progressive streams to avoid disk I/O bottleneck @@ -791,6 +804,34 @@ fun PlayerScreen( scope = coroutineScope ) } + + // Wire AudioCaptureProcessor → GeminiLiveTranslationService when live audio is active. + LaunchedEffect(uiState.isLiveAudioTranslating) { + aiRenderersFactory.audioCaptureProcessor?.onChunk = if (uiState.isLiveAudioTranslating) { + { bytes, captureMs -> viewModel.geminiLiveService.sendAudioChunk(bytes, captureMs) } + } else { + null + } + } + // Let "Find best match" read the selected reference track's buffered (upcoming) cues, + // so built-in matching can complete before dialogue is spoken. + LaunchedEffect(aiRenderersFactory) { + viewModel.bufferedReferenceIntervalsProvider = { max -> + aiRenderersFactory.extractBufferedReferenceIntervals(max) + } + viewModel.bufferedCueTextsProvider = { max -> + aiRenderersFactory.extractBufferedCueTexts(max) + } + } + DisposableEffect(aiRenderersFactory) { + onDispose { + aiRenderersFactory.audioCaptureProcessor?.onChunk = null + viewModel.bufferedReferenceIntervalsProvider = null + viewModel.bufferedCueTextsProvider = null + viewModel.geminiLiveService.disconnect() + } + } + val exoPlayer = remember(isConstrainedPlaybackDevice, preferExtensionDecoder, playbackBufferProfile) { val loadControl = DefaultLoadControl.Builder() .setBufferDurationsMs( @@ -875,6 +916,16 @@ fun PlayerScreen( } } + // Feed the selected text track's rendered cues to "Find best match" so it can + // read a built-in subtitle's timing (embedded tracks have no URL to parse). + override fun onCues(cueGroup: androidx.media3.common.text.CueGroup) { + viewModel.onPlayerCues( + cueGroup.cues.isNotEmpty(), + cueGroup.presentationTimeUs / 1000L, + cueGroup.cues.firstOrNull()?.text?.toString() + ) + } + override fun onIsPlayingChanged(playing: Boolean) { if (BuildConfig.DEBUG) { } @@ -1559,21 +1610,45 @@ fun PlayerScreen( } if (subtitle.isEmbedded && subtitle.groupIndex != null && subtitle.trackIndex != null) { - // For embedded subs, just select the track directly - val groups = exoPlayer.currentTracks.groups - val params = exoPlayer.trackSelectionParameters.buildUpon() - .clearOverridesOfType(C.TRACK_TYPE_TEXT) - .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, false) - if (subtitle.groupIndex in groups.indices && - groups[subtitle.groupIndex].type == C.TRACK_TYPE_TEXT) { - params.setOverrideForType( - androidx.media3.common.TrackSelectionOverride( - groups[subtitle.groupIndex].mediaTrackGroup, - subtitle.trackIndex - ) - ) + // Embedded subs: apply a track override. Track lists refresh asynchronously (e.g. + // right after a MediaItem rebuild), so retry briefly with freshly-resolved indices + // instead of a one-shot: a stale one-shot either selected the WRONG track via the + // preferred-language fallback (find-best-match then scored a candidate against + // itself) or, if it just disabled text, starved the scan of reference cues. + repeat(20) { attempt -> + val groups = exoPlayer.currentTracks.groups + val fresh = latestUiState.subtitles.firstOrNull { + it.isEmbedded && it.id == subtitle.id && + it.groupIndex != null && it.trackIndex != null + } ?: subtitle + val gi = fresh.groupIndex + val ti = fresh.trackIndex + val valid = gi != null && ti != null && gi in groups.indices && + groups[gi].type == C.TRACK_TYPE_TEXT && + ti < groups[gi].mediaTrackGroup.length + val params = exoPlayer.trackSelectionParameters.buildUpon() + .clearOverridesOfType(C.TRACK_TYPE_TEXT) + if (valid) { + params + .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, false) + // Clear the preferred-language fallback: with an explicit override it's + // redundant, and if the override turns invalid it silently selects a + // DIFFERENT track. + .setPreferredTextLanguage(null) + .setOverrideForType( + androidx.media3.common.TrackSelectionOverride( + groups[gi!!].mediaTrackGroup, + ti!! + ) + ) + exoPlayer.trackSelectionParameters = params.build() + return@LaunchedEffect + } + // Not ready yet: showing the wrong track is worse than showing nothing. + params.setTrackTypeDisabled(C.TRACK_TYPE_TEXT, true) + exoPlayer.trackSelectionParameters = params.build() + delay(500) } - exoPlayer.trackSelectionParameters = params.build() return@LaunchedEffect } @@ -1810,6 +1885,12 @@ fun PlayerScreen( !userSelectedSourceManually && tryAdvanceToNextStream() ) { + // Restart the startup clock immediately: the real reset happens only after + // the next stream resolves (async). Without this, the loop re-evaluates the + // stale clock on the very next iteration and fires a failover burst that + // burns through the entire source list in milliseconds. + streamSelectedTime = System.currentTimeMillis() + startupRecoverAttempted = false continue } startupHardFailureReported = true @@ -2372,10 +2453,13 @@ fun PlayerScreen( } else -> { val group = subtitleGroups.getOrNull(subtitleLangIndex - 1) + val matchGroup = latestUiState.matchLanguageName.isNotBlank() && + group?.first?.equals(latestUiState.matchLanguageName, ignoreCase = true) == true val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - val trackCount = (group?.second?.size ?: 0) + if (aiGroup) 1 else 0 + val headerCount = (if (matchGroup) 1 else 0) + (if (aiGroup) 1 else 0) + val trackCount = (group?.second?.size ?: 0) + headerCount if (subtitleTrackIndex < trackCount - 1) subtitleTrackIndex++ } } @@ -2440,12 +2524,20 @@ fun PlayerScreen( } } else { val group = subtitleGroups.getOrNull(subtitleLangIndex - 1) + val matchGroup = latestUiState.matchLanguageName.isNotBlank() && + group?.first?.equals(latestUiState.matchLanguageName, ignoreCase = true) == true val aiGroup = latestUiState.isAiAvailable && latestUiState.aiTargetLanguageName.isNotBlank() && group?.first?.equals(latestUiState.aiTargetLanguageName, ignoreCase = true) == true - val realIdx = if (aiGroup) subtitleTrackIndex - 1 else subtitleTrackIndex - if (aiGroup && subtitleTrackIndex == 0) { - if (!latestUiState.isAiTranslating) viewModel.activateAiSubtitle() + // Target-language group headers: Find Best Match first (AI-independent), + // then the AI translate entry when AI is available, then the subs. + val aiHeaderIdx = if (matchGroup) 1 else 0 + val headerCount = aiHeaderIdx + (if (aiGroup) 1 else 0) + val realIdx = subtitleTrackIndex - headerCount + if (matchGroup && subtitleTrackIndex == 0) { + viewModel.runFindBestMatch() + } else if (aiGroup && subtitleTrackIndex == aiHeaderIdx) { + if (!latestUiState.isAiTranslating) viewModel.activateAiTranslation() } else { group?.second?.getOrNull(realIdx)?.second ?.let { viewModel.selectSubtitle(it) } @@ -2652,6 +2744,16 @@ fun PlayerScreen( val pipSubScale = if (isInPipMode) 0.4f else 1f setFixedTextSize(android.util.TypedValue.COMPLEX_UNIT_SP, baseSizeSp * (subtitleSizePct / 100f) * pipSubScale) setBottomPaddingFraction((subtitleVerticalPct / 100f).coerceIn(0f, 0.5f)) + // "Find best match" without AI showing selects the built-in reference + // track under the hood to read its timing — hide its raw (e.g. English) + // cues while the scan runs. Display-only: the scan's cue collection + // listens on the player, not this view. With AI translating, the + // on-screen text is the translation, so nothing is hidden. + visibility = if (uiState.isFindingBestMatch && !uiState.isAiTranslating) { + android.view.View.INVISIBLE + } else { + android.view.View.VISIBLE + } } }, modifier = Modifier.fillMaxSize() @@ -2754,6 +2856,37 @@ fun PlayerScreen( } } + // (AI-hearing on-screen overlay removed — the hearing still runs under the hood to power + // "Find Best Match", but its transcription is no longer displayed.) + + // "Find Best Match" persistent searching indicator + if (hasPlaybackStarted && uiState.isFindingBestMatch) { + androidx.compose.foundation.layout.Row( + modifier = Modifier + .align(Alignment.TopCenter) + .padding(top = 16.dp) + .background( + color = androidx.compose.ui.graphics.Color.Black.copy(alpha = 0.72f), + shape = RoundedCornerShape(20.dp) + ) + .padding(horizontal = 16.dp, vertical = 8.dp) + .zIndex(6f), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp) + ) { + androidx.compose.material3.CircularProgressIndicator( + modifier = Modifier.size(14.dp), + strokeWidth = 2.dp, + color = androidx.compose.ui.graphics.Color(0xFF7EC8F0) + ) + Text( + text = uiState.matchStatusText.ifBlank { "Searching for a match…" }, + style = androidx.compose.material3.MaterialTheme.typography.labelLarge, + color = androidx.compose.ui.graphics.Color.White.copy(alpha = 0.9f) + ) + } + } + // AI translation API error toast uiState.aiErrorToast?.let { msg -> Toast( @@ -2765,6 +2898,17 @@ fun PlayerScreen( ) } + // "Find best match" status toast + uiState.matchToast?.let { msg -> + Toast( + message = msg, + type = ToastType.INFO, + isVisible = true, + durationMs = 4000, + onDismiss = { viewModel.dismissMatchToast() } + ) + } + // Netflix-style Controls Overlay AnimatedVisibility( visible = hasPlaybackStarted && showControls && !showSubtitleMenu && !showSourceMenu && !isInPipMode, @@ -3242,6 +3386,7 @@ fun PlayerScreen( isAiTranslating = uiState.isAiTranslating, isAiAvailable = uiState.isAiAvailable, aiTargetLanguageName = uiState.aiTargetLanguageName, + matchLanguageName = uiState.matchLanguageName, audioTracks = audioTracks, selectedAudioIndex = selectedAudioIndex, activeTab = subtitleMenuTab, @@ -3280,7 +3425,11 @@ fun PlayerScreen( try { subtitleButtonFocusRequester.requestFocus() } catch (_: Exception) {} } }, - onToggleAi = { viewModel.activateAiSubtitle() }, + isLiveAudioTranslating = uiState.isLiveAudioTranslating, + isFindingBestMatch = uiState.isFindingBestMatch, + onToggleAi = { viewModel.activateAiTranslation() }, + onToggleLiveAudio = { viewModel.toggleLiveAudioTranslation() }, + onFindBestMatch = { viewModel.runFindBestMatch() }, onClose = { showSubtitleMenu = false showControls = true @@ -4105,6 +4254,9 @@ private fun SubtitleMenu( isAiTranslating: Boolean = false, isAiAvailable: Boolean = false, aiTargetLanguageName: String = "", + matchLanguageName: String = "", + isLiveAudioTranslating: Boolean = false, + isFindingBestMatch: Boolean = false, audioTracks: List, selectedAudioIndex: Int, activeTab: Int, @@ -4118,6 +4270,8 @@ private fun SubtitleMenu( onSelectSubtitle: (Int) -> Unit, onSelectAudio: (AudioTrackInfo) -> Unit, onToggleAi: () -> Unit = {}, + onToggleLiveAudio: () -> Unit = {}, + onFindBestMatch: () -> Unit = {}, onClose: () -> Unit ) { val isMobile = LocalDeviceType.current.isTouchDevice() @@ -4207,7 +4361,7 @@ private fun SubtitleMenu( count = 0, isFocused = subtitlePanelFocus == 0 && subtitleLangIndex == 0, isActivePanel = subtitleLangIndex == 0, - isSelected = selectedSubtitle == null + isSelected = selectedSubtitle == null && !isLiveAudioTranslating ) } itemsIndexed(subtitleGroups) { idx, (langName, items) -> @@ -4216,11 +4370,10 @@ private fun SubtitleMenu( count = items.size, isFocused = subtitlePanelFocus == 0 && subtitleLangIndex == idx + 1, isActivePanel = subtitleLangIndex == idx + 1, - isSelected = if (isAiTranslating && aiTargetLanguageName.isNotBlank() && - langName.equals(aiTargetLanguageName, ignoreCase = true)) { - true - } else { - selectedSubtitle != null && + isSelected = when { + (isAiTranslating || isFindingBestMatch) && aiTargetLanguageName.isNotBlank() && + langName.equals(aiTargetLanguageName, ignoreCase = true) -> true + else -> selectedSubtitle != null && items.any { (_, sub) -> sub.id == selectedSubtitle.id } } ) @@ -4252,8 +4405,13 @@ private fun SubtitleMenu( ) } } else { + val isLiveAudioGroup = selectedGroup.first == "Live Audio" + val isMatchGroup = matchLanguageName.isNotBlank() && + selectedGroup.first.equals(matchLanguageName, ignoreCase = true) val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && selectedGroup.first.equals(aiTargetLanguageName, ignoreCase = true) + val aiHeaderIdx = if (isMatchGroup) 1 else 0 + val headerCount = aiHeaderIdx + (if (isAiGroup) 1 else 0) LazyColumn( state = trackListState, modifier = Modifier @@ -4262,14 +4420,28 @@ private fun SubtitleMenu( .padding(start = 8.dp), verticalArrangement = Arrangement.spacedBy(2.dp) ) { + if (isMatchGroup) { + // First: "Find Best Match" — timing scan, works without AI. + item { + TrackMenuItem( + label = if (isFindingBestMatch) "Scanning…" else "Find Best Match", + subtitle = "Auto", + subtitleDetail = "Auto-pick the best-synced subtitle", + isSelected = isFindingBestMatch, + isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 0, + onClick = { /* D-pad only */ } + ) + } + } if (isAiGroup) { + // Translate the built-in subtitle with AI. item { TrackMenuItem( label = aiTargetLanguageName, subtitle = "AI", subtitleDetail = null, isSelected = isAiTranslating, - isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == 0, + isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == aiHeaderIdx, onClick = { /* D-pad only */ } ) } @@ -4293,12 +4465,12 @@ private fun SubtitleMenu( .ifBlank { subtitle.id } .ifBlank { null } } - val itemIdx = if (isAiGroup) idx + 1 else idx + val itemIdx = idx + headerCount TrackMenuItem( label = mainLabel, subtitle = badge, subtitleDetail = detail, - isSelected = !isAiTranslating && selectedSubtitle?.id == subtitle.id, + isSelected = !isAiTranslating && !isLiveAudioTranslating && selectedSubtitle?.id == subtitle.id, isFocused = subtitlePanelFocus == 1 && subtitleTrackIndex == itemIdx, onClick = { /* D-pad only */ } ) @@ -4489,7 +4661,7 @@ private fun SubtitleMenu( MobileTrackItem( name = "Off", description = null, - isSelected = selectedSubtitle == null, + isSelected = selectedSubtitle == null && !isLiveAudioTranslating, onClick = { onSelectSubtitle(0) } ) Box( @@ -4503,6 +4675,9 @@ private fun SubtitleMenu( // Grouped by language subtitleGroups.forEach { (langName, indexedSubs) -> + val isLiveAudioGroup = langName == "Live Audio" + val isMatchGroup = matchLanguageName.isNotBlank() && + langName.equals(matchLanguageName, ignoreCase = true) val isAiGroup = isAiAvailable && aiTargetLanguageName.isNotBlank() && langName.equals(aiTargetLanguageName, ignoreCase = true) item(key = "mobile_header_$langName") { @@ -4518,6 +4693,40 @@ private fun SubtitleMenu( .padding(start = 16.dp, top = 8.dp, bottom = 2.dp) ) } + if (isLiveAudioGroup) { + item(key = "mobile_live_audio_item") { + MobileTrackItem( + name = "Translate Audio", + description = "AI", + isSelected = isLiveAudioTranslating, + onClick = { onToggleLiveAudio(); onClose() } + ) + Box( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 8.dp) + .height(1.dp) + .background(Color.White.copy(alpha = 0.06f)) + ) + } + } + if (isMatchGroup) { + item(key = "mobile_find_best_match_item") { + MobileTrackItem( + name = if (isFindingBestMatch) "Scanning…" else "Find Best Match", + description = "Auto", + isSelected = isFindingBestMatch, + onClick = { onFindBestMatch(); onClose() } + ) + Box( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 8.dp) + .height(1.dp) + .background(Color.White.copy(alpha = 0.06f)) + ) + } + } if (isAiGroup) { item(key = "mobile_ai_item") { MobileTrackItem( @@ -4552,7 +4761,7 @@ private fun SubtitleMenu( MobileTrackItem( name = displayName, description = description, - isSelected = !isAiTranslating && selectedSubtitle?.id == sub.id, + isSelected = !isAiTranslating && !isLiveAudioTranslating && selectedSubtitle?.id == sub.id, onClick = { onSelectSubtitle(originalIndex + 1) } ) Box( @@ -4933,7 +5142,9 @@ private fun buildExternalSubtitleConfigurations(subtitles: List): List } private fun subtitleMimeTypeFromUrl(url: String): String { - val cleanUrl = url.substringBefore('?').lowercase() + // Trailing slash before the query is real-world (AIOStreams: ".../sub.vtt/?lang=…") — trim it + // so the extension check still sees ".vtt"; the SRT fallback would silently fail on WEBVTT. + val cleanUrl = url.substringBefore('?').trimEnd('/').lowercase() return when { cleanUrl.endsWith(".vtt") -> MimeTypes.TEXT_VTT cleanUrl.endsWith(".srt") || cleanUrl.endsWith(".srt.gz") -> MimeTypes.APPLICATION_SUBRIP diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt index b42aa0c59..2a819cb8c 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt @@ -39,6 +39,7 @@ import com.google.gson.reflect.TypeToken import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -47,8 +48,11 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import javax.inject.Inject private fun isSupplementalStream(stream: StreamSource): Boolean = @@ -124,6 +128,16 @@ data class PlayerUiState( val aiTargetLanguageName: String = "", // Non-null while an AI translation API error toast should be visible val aiErrorToast: String? = null, + // True while Gemini Live audio translation is active for this session + val isLiveAudioTranslating: Boolean = false, + // True while "Find best match" is scanning subtitles; message shown as a transient toast. + val isFindingBestMatch: Boolean = false, + val matchToast: String? = null, + // Persistent status shown on screen for the whole duration of a "Find best match" scan. + val matchStatusText: String = "", + // Full name of the preferred subtitle language (e.g. "Hebrew") — drives the "Find Best Match" + // menu entry. Independent of AI availability: the timing scan needs no AI/API key. + val matchLanguageName: String = "", // Episode title for TV shows (e.g. "The Devil's Verdict"), populated from TMDB season details val episodeTitle: String? = null, // Plot synopsis from TMDB, used in the pause overlay metadata block @@ -185,11 +199,24 @@ class PlayerViewModel @Inject constructor( private var lastIsPlaying: Boolean = false private var hasMarkedWatched: Boolean = false private var hasManualSubtitleSelection: Boolean = false + // True only when the user explicitly picked a subtitle track from the menu (not AI/auto-match). + // A late-arriving embedded preferred-language track overrides auto selections but never this. + private var userPickedSubtitle: Boolean = false private var playbackSessionStartTime: Long = 0L // AI subtitle settings (read once per video load) private var aiSubtitleEnabled = false private var aiSubtitleAutoSelect = false + // When true (default), activating AI first tries to auto-pick a synced addon subtitle in the + // user's preferred language (Find Best Match); only if nothing matches does it translate the + // built-in subtitle. + private var aiFindBestMatchFirst = false + // Normalized code of the user's preferred subtitle language (e.g. "he") — the target of both + // "find best match" and AI translation. Blank when unset/disabled. + @Volatile private var targetSubtitleLangCode = "" + // One auto-run of "find best match" per playback/stream — applyPreferredSubtitle re-runs on + // every subtitle-list update and must not restart a completed scan. + private var autoMatchAttempted = false private var aiApiKey = "" private var aiModel = SubtitleAiModel.GROQ_LLAMA_70B private var aiRemoveHearingImpaired = true @@ -197,9 +224,13 @@ class PlayerViewModel @Inject constructor( // Global AI subtitle DataStore keys (device-wide, not profile-scoped) private val aiEnabledKey = booleanPreferencesKey("subtitle_ai_enabled") private val aiAutoSelectKey = booleanPreferencesKey("subtitle_ai_auto_select") + private val aiFindBestMatchKey = booleanPreferencesKey("subtitle_ai_find_best_match") private val aiApiKeyKey = globalStringPreferencesKey("subtitle_ai_api_key") private val aiModelKey = globalStringPreferencesKey("subtitle_ai_model") private val aiRemoveHearingImpairedKey = booleanPreferencesKey("subtitle_remove_hearing_impaired") + // Remembered "find best match" results, keyed by stream identity (sync is a property of the + // exact video file, not the title — a different rip needs a fresh scan). + private val subtitleMatchCacheKey = globalStringPreferencesKey("subtitle_match_cache_v1") private val _isTranslatingLive = MutableStateFlow(false) val isTranslatingLive: kotlinx.coroutines.flow.StateFlow = _isTranslatingLive.asStateFlow() @@ -233,6 +264,69 @@ class PlayerViewModel @Inject constructor( } } + val geminiLiveService: GeminiLiveTranslationService = GeminiLiveTranslationService( + apiKeyProvider = { aiApiKey }, + scope = viewModelScope + ) + val liveAudioText = geminiLiveService.translatedText + val liveAudioState = geminiLiveService.state + + // Latest playback media position, used to time-align AI-hearing samples with subtitle cues. + @Volatile private var lastKnownPositionMs = 0L + + private var findMatchJob: Job? = null + + // Built-in reference collection: while active, rendered cues of the selected (built-in English) + // track are turned into cue-visible intervals used as the sync reference. + @Volatile private var collectingReference = false + @Volatile private var refIntervalStart = -1L + private val referenceIntervals = mutableListOf>() + + /** Set by the player screen: reads the selected text track's already-buffered cue intervals. */ + @Volatile var bufferedReferenceIntervalsProvider: ((Int) -> List>)? = null + + /** Set by the player screen: reads the selected text track's already-buffered cue texts. */ + @Volatile var bufferedCueTextsProvider: ((Int) -> List)? = null + + // Normalized lines of the candidate displayed when the scan started — set for the duration of + // reference collection so BOTH the buffered and the realtime paths can reject self-cues (the + // reference track switch landing late/never would otherwise score a candidate against itself). + @Volatile private var matchPreviousTexts: Set? = null + @Volatile private var refIntervalSelfText = false + + /** Rendered-cue callback from the player; records reference intervals during a built-in scan. */ + fun onPlayerCues(hasText: Boolean, timeMs: Long, textSample: String? = null) { + if (!collectingReference || timeMs < 0) return + if (hasText) { + if (refIntervalStart < 0) { + refIntervalStart = timeMs + refIntervalSelfText = false + } + val prev = matchPreviousTexts + if (prev != null && textSample != null && + normalizeCueTextForCompare(textSample) in prev + ) { + refIntervalSelfText = true + } + } else if (refIntervalStart in 0 until timeMs) { + // An interval spanning a forward seek would record a bogus minutes-long "cue"; + // real cues never stay on screen this long — drop it. + val validLength = timeMs - refIntervalStart <= MATCH_MAX_REF_INTERVAL_MS + if (validLength && !refIntervalSelfText) { + synchronized(referenceIntervals) { referenceIntervals.add(refIntervalStart to timeMs) } + } else if (refIntervalSelfText) { + android.util.Log.w( + "SubMatch", + "realtime self-cue interval dropped ${refIntervalStart}..$timeMs — reference track switch not landed?" + ) + } + refIntervalStart = -1L + } else if (refIntervalStart >= 0) { + // Position jumped backwards past the interval start (seek) — discard the open interval. + refIntervalStart = -1L + } + } + // Skip intro private var skipIntervals: List = emptyList() private var lastActiveSkipType: String? = null @@ -340,6 +434,10 @@ class PlayerViewModel @Inject constructor( lastWatchHistorySavedPositionSeconds = -1L subtitleRefreshJob?.cancel() subtitleSelectionJob?.cancel() + // Cancel any in-flight "Find best match" scan from the previous title — otherwise it can + // finish on this new session and select/restore a stale subtitle or poison the match cache + // (its stream cache key resolves to the new stream). Same-VM path only, e.g. play-next. + cancelFindBestMatch() vodAppendJob?.cancel() homeServerAppendJob?.cancel() streamPrewarmJob?.cancel() @@ -348,6 +446,8 @@ class PlayerViewModel @Inject constructor( playbackErrorReportJob?.cancel() primaryStreamResolutionFinal = false hasManualSubtitleSelection = false + userPickedSubtitle = false + autoMatchAttempted = false aiSourceSubtitle = null val cachedItem = mediaRepository.getCachedItem(mediaType, mediaId) val cachedLogoUrl = mediaRepository.peekCachedLogoUrl(mediaType, mediaId) @@ -355,6 +455,10 @@ class PlayerViewModel @Inject constructor( isLoading = true, isLoadingStreams = false, selectedSubtitle = null, + // A new video means a new subtitle universe — stale externals from the previous + // title poisoned both the picker and the match scan (references from the new video + // scored against the old video's candidates). + subtitles = emptyList(), isAiTranslating = false, isAiAvailable = false, aiTargetLanguageName = "", @@ -399,10 +503,12 @@ class PlayerViewModel @Inject constructor( .let { if (isSubtitleDisabledPreference(it)) "" else it } val secondarySub = prefs[secondarySubtitleKey()]?.trim().orEmpty() .let { if (isSubtitleDisabledPreference(it)) "" else it } + setTargetSubtitleLang(normalizeLanguage(preferredSub)) // Load AI subtitle settings aiSubtitleEnabled = prefs[aiEnabledKey] ?: false aiSubtitleAutoSelect = prefs[aiAutoSelectKey] ?: false + aiFindBestMatchFirst = prefs[aiFindBestMatchKey] ?: false aiApiKey = prefs[aiApiKeyKey] ?: "" aiModel = runCatching { SubtitleAiModel.valueOf(prefs[aiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) @@ -497,58 +603,85 @@ class PlayerViewModel @Inject constructor( selectedStreamUrl = resolvedProvidedUrl, savedPosition = resumeData.positionMs ) + // NOTE: these background children share the load job — an uncaught exception in + // any of them cancels ALL siblings (that silently killed the subtitle flow). + // Every body is therefore failure-isolated and logged. + fun childFailed(name: String): (Throwable) -> Unit = { e -> + if (e !is kotlinx.coroutines.CancellationException) { + android.util.Log.w("PlayerViewModel", "load child '$name' failed: ${e.message}") + } + } launch { kotlinx.coroutines.delay(1_500L) - populateStreamsForProvidedUrl( - mediaType = mediaType, - mediaId = mediaId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - providedImdbId = providedImdbId, - playbackUrl = resolvedProvidedUrl - ) + runCatching { + populateStreamsForProvidedUrl( + mediaType = mediaType, + mediaId = mediaId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + providedImdbId = providedImdbId, + playbackUrl = resolvedProvidedUrl + ) + }.onFailure(childFailed("populateStreams")) } // Load IPTV and home-server sources from cache in parallel so the // source picker in the player shows alternatives immediately. homeServerAppendJob?.cancel() homeServerAppendJob = launch { - appendHomeServerSourcesInBackground( - mediaType = mediaType, - imdbId = currentImdbId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - timeoutMs = 5_000L - ) + runCatching { + appendHomeServerSourcesInBackground( + mediaType = mediaType, + imdbId = currentImdbId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + timeoutMs = 5_000L + ) + }.onFailure(childFailed("homeServerAppend")) } vodAppendJob?.cancel() vodAppendJob = launch { - appendVodSourceInBackground( - mediaType = mediaType, - imdbId = currentImdbId, - seasonNumber = seasonNumber, - episodeNumber = episodeNumber, - timeoutMs = 15_000L - ) + runCatching { + appendVodSourceInBackground( + mediaType = mediaType, + imdbId = currentImdbId, + seasonNumber = seasonNumber, + episodeNumber = episodeNumber, + timeoutMs = 15_000L + ) + }.onFailure(childFailed("vodAppend")) } // Fetch metadata in background - launch { fetchMediaMetadata(mediaType, mediaId) } + launch { + runCatching { fetchMediaMetadata(mediaType, mediaId) } + .onFailure(childFailed("metadata")) + } // Fetch skip intervals in background (needs IMDB id) launch { - if (mediaType == MediaType.TV && seasonNumber != null && episodeNumber != null) { - val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) - val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId - if (!imdbId.isNullOrBlank()) { - currentImdbId = imdbId - if (cachedImdbId == null) mediaRepository.cacheImdbId(mediaType, mediaId, imdbId) - fetchSkipIntervals(imdbId, seasonNumber, episodeNumber) + runCatching { + if (mediaType == MediaType.TV && seasonNumber != null && episodeNumber != null) { + val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) + val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId + if (!imdbId.isNullOrBlank()) { + currentImdbId = imdbId + if (cachedImdbId == null) mediaRepository.cacheImdbId(mediaType, mediaId, imdbId) + fetchSkipIntervals(imdbId, seasonNumber, episodeNumber) + } } - } + }.onFailure(childFailed("skipIntervals")) } // Direct-URL playback must still fetch subtitle addons (e.g. OpenSubtitles). - launch { + // On viewModelScope (NOT a child of this load job): sibling children above + // (VOD/home-server/stream appends) occasionally throw, which cancels the whole + // load job and — because this block is registered last — killed the subtitle + // flow before its first line ran: no fetch, no selection, playback with + // subtitles silently Off. loadMedia cancels subtitleRefreshJob, so lifecycle + // stays correct across videos. + subtitleRefreshJob?.cancel() + subtitleRefreshJob = viewModelScope.launch { _uiState.value = _uiState.value.copy(isLoadingSubtitles = true) val cachedImdbId = currentImdbId ?: mediaRepository.getCachedImdbId(mediaType, mediaId) - val imdbId = cachedImdbId ?: resolveExternalIds(mediaType, mediaId).imdbId + val imdbId = cachedImdbId + ?: runCatching { resolveExternalIds(mediaType, mediaId).imdbId }.getOrNull() if (!imdbId.isNullOrBlank()) { currentImdbId = imdbId @@ -578,7 +711,15 @@ class PlayerViewModel @Inject constructor( scheduleSubtitleSelection(currentOriginalLanguage) } else { + // No IMDb id → addon subs can't be fetched, but the selection flow must + // still run: embedded tracks and the AI option exist regardless. Silently + // stopping here left playback with NOTHING selected ("Off") and no scan. + android.util.Log.w( + "SubMatch", + "subtitle fetch skipped: no imdbId for $mediaType/$mediaId — scheduling selection with embedded-only" + ) _uiState.value = _uiState.value.copy(isLoadingSubtitles = false) + scheduleSubtitleSelection(currentOriginalLanguage) } } return@launch @@ -1042,6 +1183,7 @@ class PlayerViewModel @Inject constructor( } fun onPlaybackPosition(positionMs: Long) { + lastKnownPositionMs = positionMs updateActiveSkipInterval(positionMs) } @@ -1159,14 +1301,53 @@ class PlayerViewModel @Inject constructor( } } + /** Keeps [targetSubtitleLangCode] and the menu-facing [PlayerUiState.matchLanguageName] in sync. */ + private fun setTargetSubtitleLang(code: String) { + targetSubtitleLangCode = code + val name = if (code.isBlank()) "" else languageCodeToName(code) + if (_uiState.value.matchLanguageName != name) { + _uiState.value = _uiState.value.copy(matchLanguageName = name) + } + } + private fun applyPreferredSubtitle(preference: String, subtitles: List, fallbackLanguage: String?) { + val normalizedPref = normalizeLanguage(preference) + if (normalizedPref.isNotBlank() && !isSubtitleDisabledPreference(preference)) { + setTargetSubtitleLang(normalizedPref) + } + + // Highest priority: an embedded (muxed) track in the preferred language is inherently in + // sync, so it beats any auto logic — it overrides an auto-selected/auto-matched subtitle and + // cancels a running "Find best match" scan. It never overrides a real user pick. This also + // handles embedded tracks that resolve *after* the auto-match already started. + if (!userPickedSubtitle && normalizedPref.isNotBlank() && !isSubtitleDisabledPreference(preference)) { + val embeddedPref = subtitles.firstOrNull { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) && + (normalizeLanguage(it.lang) == normalizedPref || normalizeLanguage(it.label) == normalizedPref) + } + if (embeddedPref != null && _uiState.value.selectedSubtitle?.id != embeddedPref.id) { + cancelFindBestMatch() + hasManualSubtitleSelection = true + translationManager.isEnabled = false + aiSourceSubtitle = null + _uiState.value = _uiState.value.copy( + selectedSubtitle = embeddedPref, + isAiTranslating = false, + isAiAvailable = false, + aiTargetLanguageName = "", + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + return + } + } + if (hasManualSubtitleSelection) return if (isSubtitleDisabledPreference(preference)) { _uiState.value = _uiState.value.copy(selectedSubtitle = null) return } - val normalizedPref = normalizeLanguage(preference) val normalizedFallback = fallbackLanguage ?.let { normalizeLanguage(it) } ?.takeIf { it.isNotBlank() && it != normalizedPref } @@ -1254,10 +1435,20 @@ class PlayerViewModel @Inject constructor( } else if (aiModeActive) { // No embedded preferred-language subtitle → activate AI (addon subs are ignored) val source = findAiSourceSubtitle(subtitles) + val targetLangName = languageCodeToName(normalizedPref) if (source != null) { - val targetLangName = languageCodeToName(normalizedPref) aiSourceSubtitle = source translationManager.targetLanguage = targetLangName + } + if (aiFindBestMatchFirst) { + // Auto-run "Find best match" first: pick a synced Hebrew addon sub if one exists, + // otherwise fall back to AI-translating the built-in source (handled inside). + _uiState.value = _uiState.value.copy( + isAiAvailable = source != null, + aiTargetLanguageName = if (source != null) targetLangName else _uiState.value.aiTargetLanguageName + ) + activateAiSubtitle() + } else if (source != null) { translationManager.isEnabled = true translationManager.reset() aiErrorToastShown = false @@ -1268,7 +1459,7 @@ class PlayerViewModel @Inject constructor( ?: normalizedFallback?.let { bestMatch(it) } if (externalMatch != null) { val current = _uiState.value.selectedSubtitle - if (current == null || !(current.isEmbedded && !externalMatch.isEmbedded)) { + if (!embeddedBlocksExternal(current, externalMatch, normalizedPref)) { translationManager.isEnabled = false aiSourceSubtitle = null _uiState.value = _uiState.value.copy(selectedSubtitle = externalMatch, isAiTranslating = false, isAiAvailable = false, aiTargetLanguageName = "") @@ -1291,7 +1482,7 @@ class PlayerViewModel @Inject constructor( ?: normalizedFallback?.let { bestMatch(it) } if (externalMatch != null) { val current = _uiState.value.selectedSubtitle - if (current == null || !(current.isEmbedded && !externalMatch.isEmbedded)) { + if (!embeddedBlocksExternal(current, externalMatch, normalizedPref)) { translationManager.isEnabled = false if (aiEnabledForLanguage) { // Keep AI available in picker — user can still activate it manually @@ -1302,9 +1493,35 @@ class PlayerViewModel @Inject constructor( } } } + // "Find best match first" auto-runs the scan even without AI auto-select: it verifies + // sync on top of the classic release-name pick. AI translation is deliberately NOT + // auto-activated here — that's what the auto-select toggle controls — and the raw + // reference cues stay hidden while scanning, so on no-match the classic pick above + // simply returns to screen. + if (aiFindBestMatchFirst && !autoMatchAttempted && !hasManualSubtitleSelection) { + autoMatchAttempted = true + findBestSubtitleMatch() + } } } + /** + * Whether a currently-selected embedded track should block auto-selecting [externalMatch]. + * An embedded track normally wins over externals (guaranteed sync) — but a fallback-language + * embedded (e.g. English selected while the preferred-language addon subs were still + * loading) must not block the preferred-language external that arrives later. + */ + private fun embeddedBlocksExternal( + current: Subtitle?, + externalMatch: Subtitle, + normalizedPref: String + ): Boolean { + if (current == null || !current.isEmbedded || externalMatch.isEmbedded) return false + val externalIsPref = normalizeLanguage(externalMatch.lang) == normalizedPref + val currentIsPref = normalizeLanguage(current.lang) == normalizedPref + return !(externalIsPref && !currentIsPref) + } + private fun findAiSourceSubtitle(subtitles: List): Subtitle? { // Some files label forced tracks "SUBFORCED" without setting SELECTION_FLAG_FORCED fun Subtitle.isEffectivelyForced() = isForced || label.contains("forced", ignoreCase = true) @@ -1318,11 +1535,13 @@ class PlayerViewModel @Inject constructor( ?: embedded.firstOrNull { it.isUsableSource() } } - // Prefer English embedded > any embedded with lang > any embedded > any non-forced subtitle + // Prefer English embedded > any embedded with lang > any embedded > external English. + // External subs in other languages are never a sane source: their timing is unverified + // and a target-language addon sub as "source" means translating a language to itself. return subtitles.filter { normalizeLanguage(it.lang) == "en" }.bestEmbedded() ?: subtitles.filter { it.lang.isNotBlank() }.bestEmbedded() ?: subtitles.bestEmbedded() - ?: subtitles.firstOrNull { it.isUsableSource() } + ?: subtitles.firstOrNull { it.isUsableSource() && normalizeLanguage(it.lang) == "en" } } private fun languageCodeToName(code: String): String { @@ -1710,20 +1929,37 @@ class PlayerViewModel @Inject constructor( ) } + /** Aggregator addons (AIOStreams) sort results server-side per the user's own web config — + * their streams keep the order the addon returned them in instead of being re-sorted. */ + private fun keepsOwnStreamOrder(stream: StreamSource): Boolean = + stream.addonName.contains("aiostream", ignoreCase = true) || + stream.addonId.contains("aiostream", ignoreCase = true) + private fun sortStreamsByQualityAndSize( streams: List, preferredLanguage: String ): List { - return streams.sortedWith( - compareBy { addonOrderIndex(it) } - .thenByDescending { parseSize(it.size) } - .thenByDescending { qualityScore(it.quality) } - .thenByDescending { playbackPriorityScore(it) } - .thenBy { streamRepository.getPlaybackHostHealthPenalty(it) } - .thenBy { if (it.behaviorHints?.notWebReady == true) 1 else 0 } - .thenByDescending { streamLanguageScore(it, preferredLanguage) } - .thenBy { it.source.lowercase() } - ) + val qualityOrder = compareByDescending> { parseSize(it.value.size) } + .thenByDescending { qualityScore(it.value.quality) } + .thenByDescending { playbackPriorityScore(it.value) } + .thenBy { streamRepository.getPlaybackHostHealthPenalty(it.value) } + .thenBy { if (it.value.behaviorHints?.notWebReady == true) 1 else 0 } + .thenByDescending { streamLanguageScore(it.value, preferredLanguage) } + .thenBy { it.value.source.lowercase() } + return streams.withIndex() + .sortedWith( + // Streams stay grouped by addon order; within a group, self-ordered addons keep + // their original (arrival) order while everything else gets the quality sort. + compareBy> { addonOrderIndex(it.value) } + .then { a, b -> + if (keepsOwnStreamOrder(a.value) && keepsOwnStreamOrder(b.value)) { + a.index.compareTo(b.index) + } else { + qualityOrder.compare(a, b) + } + } + ) + .map { it.value } } fun prewarmStream(stream: StreamSource) { @@ -2054,12 +2290,30 @@ class PlayerViewModel @Inject constructor( } } + // A different source is a different file: the previous stream's track selection, AI + // translation state, any running match scan, and — crucially — its embedded subtitle + // entries are all meaningless for it. Drop them so the auto-selection flow (find best + // match / AI) runs fresh: a stale embedded entry would otherwise be re-selected against + // the stale list and map onto an arbitrary track of the new file, while the fresh + // embedded tracks arriving later would be ignored because the selection "already + // matches" (no nonce bump → the override is never applied to the new media item). + cancelFindBestMatch() + hasManualSubtitleSelection = false + userPickedSubtitle = false + autoMatchAttempted = false + aiSourceSubtitle = null + translationManager.isEnabled = false + // Direct URL - use immediately (ExoPlayer handles redirects) _uiState.value = _uiState.value.copy( selectedStream = resolvedStream, selectedStreamUrl = url, savedPosition = requestedResumePosition ?: _uiState.value.savedPosition, streamSelectionNonce = _uiState.value.streamSelectionNonce + 1, + subtitles = _uiState.value.subtitles.filterNot { it.isEmbedded }, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1, isLoading = false, isLoadingStreams = false, streamProgress = null, @@ -2239,15 +2493,22 @@ class PlayerViewModel @Inject constructor( } val resolvedSelected = if (selected != null) { - finalList.firstOrNull { it.id == selected.id } - ?: finalList.firstOrNull { selected.url.isNotBlank() && it.url == selected.url } - // After MediaItem rebuild groupIndex can change, making generated IDs stale. - // Fall back to lang+label match for embedded subs so the nonce-based - // LaunchedEffect fires with the freshly-indexed version. - ?: if (selected.isEmbedded) finalList.firstOrNull { - it.isEmbedded && it.lang == selected.lang && it.label == selected.label - } else null - ?: selected + if (!selected.isEmbedded && selected.url.startsWith("file:")) { + // A matched subtitle localized to a cache file — keep it as-is. Remapping to + // the addon-list entry (same id) would swap the URL back to the remote server + // and re-trigger a MediaItem rebuild with the slow remote source. + selected + } else { + finalList.firstOrNull { it.id == selected.id } + ?: finalList.firstOrNull { selected.url.isNotBlank() && it.url == selected.url } + // After MediaItem rebuild groupIndex can change, making generated IDs stale. + // Fall back to lang+label match for embedded subs so the nonce-based + // LaunchedEffect fires with the freshly-indexed version. + ?: if (selected.isEmbedded) finalList.firstOrNull { + it.isEmbedded && it.lang == selected.lang && it.label == selected.label + } else null + ?: selected + } } else { null } @@ -2288,8 +2549,13 @@ class PlayerViewModel @Inject constructor( } } - fun selectSubtitle(subtitle: Subtitle) { + fun selectSubtitle(subtitle: Subtitle, isUserAction: Boolean = true) { hasManualSubtitleSelection = true + userPickedSubtitle = isUserAction + if (isUserAction) { + cancelFindBestMatch() + updateMatchCacheForManualPick(subtitle) + } subtitleSelectionJob?.cancel() translationManager.isEnabled = false // Keep isAiAvailable/aiTargetLanguageName so the AI entry stays in the menu for re-selection @@ -2301,9 +2567,112 @@ class PlayerViewModel @Inject constructor( recordSubtitleUsage(subtitle) } + /** Cancel a running/queued "Find best match" scan and clear its transient state. */ + private fun cancelFindBestMatch() { + findMatchJob?.cancel() + collectingReference = false + matchPreviousTexts = null + if (_uiState.value.isFindingBestMatch || _uiState.value.isLiveAudioTranslating) { + geminiLiveService.disconnect() + _uiState.value = _uiState.value.copy( + isFindingBestMatch = false, + isLiveAudioTranslating = false, + matchStatusText = "" + ) + } + } + + /** + * The AI subtitle entry point (menu option + auto-select). When "find best match first" is on, + * it first tries to auto-pick a synced addon subtitle in the user's preferred language; only + * if nothing matches does it fall back to AI-translating the built-in subtitle. + */ fun activateAiSubtitle() { - val source = aiSourceSubtitle ?: return + if (aiFindBestMatchFirst) { + runFindBestMatch(isUserAction = false) + } else { + activateAiTranslation() + } + } + + /** + * The "Find Best Match (AI)" menu button: always run the match logic; if nothing synced is + * found, fall back to the AI translation option when one is available. + */ + fun runFindBestMatch(isUserAction: Boolean = true) { + // The timing-based scan is AI-independent — no aiSubtitleEnabled gate. Only the optional + // AI interim below and the hearing fallback (gated at the scoring dispatch) need AI. + // A user-triggered rescan means the current (possibly remembered) pick is unwanted — + // forget the cached entry and scan fresh; the auto-run keeps using the cache. + if (isUserAction) writeCachedMatch(null) else autoMatchAttempted = true + // With AI auto-select on, show AI translation immediately (if a source exists) so the + // user sees subtitles right away — and because the AI source (English) track is then + // selected under the hood, the match can read its cue timing without ever putting English + // on screen. Then run the match in the background and upgrade to a synced addon sub if + // one is found; otherwise stay on AI. + // With AI auto-select OFF (or the AI feature disabled), the user opted out of automatic + // AI translation entirely: the scan runs "silent" (reference cues hidden, zero AI/API + // usage) and falls back to whatever was selected before. + val source = if (aiSubtitleEnabled && aiSubtitleAutoSelect) { + findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle + } else { + null + } + if (source != null) { + activateAiTranslation() + findBestSubtitleMatch( + onNoMatch = { score -> + // Staying on the AI fallback — re-activate to upgrade the translation source: + // an embedded track may have resolved while the scan ran, and the early + // activation could still be translating an external fallback source. Say so — + // otherwise the searching indicator just vanishes with no visible outcome. + if (_uiState.value.isAiTranslating) { + activateAiTranslation() + showMatchToast( + "No well-synced subtitle found" + + (score?.let { " (best ${(it * 100).toInt()}%)" } ?: "") + + " — keeping AI translation" + ) + } + }, + useCache = !isUserAction + ) + } else { + // default onNoMatch: "no well-synced subtitle" toast + findBestSubtitleMatch(useCache = !isUserAction) + } + } + + /** + * Scan-failure fallback: activate AI translation when the feature can run (master toggle on, + * key present, source available). Deliberately NOT gated on auto-select — that setting only + * governs unprompted activation at playback start; a failed scan explicitly asked for the + * best available subtitle, and AI timing (from the built-in track) beats an unverified addon + * pick. Returns true when AI took over. + */ + private fun tryAiFallbackAfterScan(): Boolean { + if (!aiSubtitleEnabled || aiApiKey.isBlank()) return false + if (findAiSourceSubtitle(_uiState.value.subtitles) == null) return false + activateAiTranslation() + showMatchToast("No well-synced subtitle found — using AI translation") + return true + } + + /** Existing behavior: translate the built-in/source subtitle to the target language. */ + fun activateAiTranslation() { + // Defense-in-depth: every caller is already gated on the AI master toggle (menu entry via + // isAiAvailable, interim via auto-select, fallback via tryAiFallbackAfterScan), but AI + // must never activate — and never spend API requests — when the feature is off. + if (!aiSubtitleEnabled) return + // Re-resolve the source on every activation: an early activation (before embedded tracks + // resolve) may have cached an external fallback source, and translation inherits the + // source's timing — keep the cached one only when nothing better is available now. + val source = findAiSourceSubtitle(_uiState.value.subtitles) ?: aiSourceSubtitle ?: return + aiSourceSubtitle = source hasManualSubtitleSelection = true + // AI activation is not a specific user track pick — a late embedded preferred-language + // track may still displace it. + userPickedSubtitle = false val targetLangName = _uiState.value.aiTargetLanguageName if (targetLangName.isNotBlank()) translationManager.targetLanguage = targetLangName translationManager.isEnabled = true @@ -2317,6 +2686,615 @@ class PlayerViewModel @Inject constructor( ) } + private var subtitleBeforeLiveAudio: Subtitle? = null + + fun toggleLiveAudioTranslation() { + if (_uiState.value.isLiveAudioTranslating) { + geminiLiveService.disconnect() + // Restore whatever subtitle was active before live AI took over + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = false, + selectedSubtitle = subtitleBeforeLiveAudio, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + subtitleBeforeLiveAudio = null + } else { + // Clear current subtitle so ExoPlayer stops rendering it while live AI is active. + // Must mark this as a manual selection — otherwise a subtitle-fetch job that + // resolves after this point (scheduleSubtitleSelection) would silently re-select + // a subtitle on top of the live AI overlay, since it only checks this flag. + hasManualSubtitleSelection = true + subtitleSelectionJob?.cancel() + translationManager.isEnabled = false + subtitleBeforeLiveAudio = _uiState.value.selectedSubtitle + targetSubtitleLangCode.takeIf { it.isNotBlank() }?.let { geminiLiveService.targetLanguageCode = it } + geminiLiveService.connect() + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = true, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + } + + /** + * "Find best match": scan the addon subtitle tracks in the user's preferred language and + * auto-select the one that is best synced with the audio. Loads every candidate's cues, + * listens to the AI hearing for a short window, then scores each candidate by how often its + * on-screen cue matches what was spoken at that moment (see [SubtitleSyncMatcher]). + */ + fun findBestSubtitleMatch(onNoMatch: ((Double?) -> Unit)? = null, useCache: Boolean = true) { + if (_uiState.value.isFindingBestMatch) return + val previousSubtitle = _uiState.value.selectedSubtitle + findMatchJob?.cancel() + findMatchJob = viewModelScope.launch { + val targetLang = targetSubtitleLangCode.ifBlank { normalizeLanguage(getDefaultSubtitle()) } + val targetLangName = languageCodeToName(targetLang) + // Include the best (rejected) score so a fast verdict is visibly a real scan result. + val noMatch = onNoMatch ?: { score -> + showMatchToast( + "No well-synced $targetLangName subtitle found" + + (score?.let { " (best ${(it * 100).toInt()}%)" } ?: "") + ) + } + if (targetLang.isBlank() || isSubtitleDisabledPreference(targetLang)) { + noMatch(null) + return@launch + } + beginMatch("Finding best subtitle…") + + // Subtitle sources resolve asynchronously after playback starts: embedded tracks a beat + // later, addon subtitles when their fetch completes. Wait for both a usable muxed + // reference and target-language addon candidates — otherwise the scan can fire before + // the addon list exists and silently find nothing. (AI translation is already showing + // meanwhile, so waiting here costs the user nothing.) + val waitStart = System.currentTimeMillis() + var playingSinceMs = -1L + while (true) { + val nowMs = System.currentTimeMillis() + val elapsed = nowMs - waitStart + // Grace periods count *continuous playback* time, not wall time: embedded tracks + // cannot resolve before the stream actually plays, so a scan that gives up while + // the player is still buffering (e.g. right after a source switch) is meaningless. + if (lastIsPlaying) { + if (playingSinceMs < 0) playingSinceMs = nowMs + } else { + playingSinceMs = -1L + } + val playingFor = if (playingSinceMs < 0) -1L else nowMs - playingSinceMs + val current = _uiState.value.subtitles + val hasEmbedded = current.any { it.isEmbedded && !it.isBitmap && !it.isForced } + val hasCandidates = current.any { + !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && + normalizeLanguage(it.lang) == targetLang + } + if (hasEmbedded && hasCandidates) break + if (playingFor >= MATCH_SOURCES_WAIT_MS) break + // Past the embedded grace period and the addon fetch is done — whatever is missing + // now isn't coming; proceed with what we have. + if (playingFor >= MATCH_EMBEDDED_WAIT_MS && !_uiState.value.isLoadingSubtitles) break + // Stream never started (stall/error) — stop waiting eventually. + if (elapsed >= MATCH_PLAYBACK_WAIT_CAP_MS) break + delay(200) + } + val subs = _uiState.value.subtitles + + // An embedded target-language track is muxed into the media → guaranteed in sync. + val embedded = subs.firstOrNull { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) && + normalizeLanguage(it.lang) == targetLang + } + if (embedded != null) { + endMatch() + selectSubtitle(embedded, isUserAction = false) + showMatchToast("Matched: embedded $targetLangName subtitle (in sync)") + return@launch + } + + // Cap the number of candidates we download/score — addons can return dozens. Rank by + // the release-name heuristic first so the subs most likely cut for this rip survive. + val streamSrc = _uiState.value.selectedStream?.source.orEmpty() + val candidates = subs.filter { + !it.isEmbedded && !it.isBitmap && it.url.isNotBlank() && + normalizeLanguage(it.lang) == targetLang + } + .sortedByDescending { weightedSubtitleScore(streamSrc, it.id) } + .take(MATCH_MAX_CANDIDATES) + if (candidates.isEmpty()) { + endMatch() + // No candidates at all in the target language — AI translation is the only + // possible target-language subtitle; fall back to it when available. + if (!_uiState.value.isAiTranslating && !tryAiFallbackAfterScan()) noMatch(null) + return@launch + } + + // A previously matched subtitle for this exact stream (same file → same sync) that is + // still offered by the addons wins immediately — no need to rescan. Skipped for + // user-triggered rescans, which mean the remembered pick is unwanted. + if (useCache) { + val remembered = readCachedMatch()?.let { cached -> + candidates.firstOrNull { it.id == cached.id && it.provider == cached.provider } + } + if (remembered != null) { + // Serve from a local cache file so the MediaItem rebuild doesn't stall on a + // slow addon server (download bounded by the matcher's client timeouts). + val local = SubtitleSyncMatcher.loadRaw(remembered.url) + ?.let { localizeSubtitle(remembered, it) } ?: remembered + endMatch() + selectSubtitle(local, isUserAction = false) + showMatchToast("Matched: ${remembered.label} (remembered)") + return@launch + } + } + + // Prefer any embedded (muxed) track as the sync reference (English first, else any). + // SDH/CC is fine for *timing*; only forced/bitmap are unsuitable. + val embeddedRefs = subs.filter { + it.isEmbedded && !it.isBitmap && !it.isForced && + !it.label.contains("forced", ignoreCase = true) + } + val builtInReference = embeddedRefs.firstOrNull { normalizeLanguage(it.lang) == "en" } + ?: embeddedRefs.firstOrNull() + val sourceLabel = if (builtInReference != null) "Built-in" else "Hearing" + android.util.Log.i( + "SubMatch", + "reference source=$sourceLabel embeddedRefs=${embeddedRefs.size} " + + "ref=\"${builtInReference?.label ?: "-"}\" (lang=${builtInReference?.lang}) " + + "allEmbedded=${subs.count { it.isEmbedded }}" + ) + updateMatchStatus("Finding best subtitle ($sourceLabel)…") + + // Keep the raw text alongside the parsed cues: the winning subtitle is later served to + // ExoPlayer from a local cache file (already downloaded here) instead of re-fetching + // a possibly slow addon server during the MediaItem rebuild. + val loadedRaw = candidates.map { sub -> + async { sub to SubtitleSyncMatcher.loadRaw(sub.url) } + }.awaitAll() + val rawBySubKey = loadedRaw.mapNotNull { (sub, raw) -> + raw?.let { "${sub.provider}|${sub.id}" to it } + }.toMap() + val loaded = loadedRaw.mapNotNull { (sub, raw) -> + val cues = raw?.let { SubtitleSyncMatcher.parseCues(it) }.orEmpty() + if (cues.isEmpty()) null else sub to cues + } + + /** Select [sub], serving it from a local file when its text was downloaded. */ + fun selectServedLocally(sub: Subtitle) { + val localized = rawBySubKey["${sub.provider}|${sub.id}"] + ?.let { localizeSubtitle(sub, it) } ?: sub + selectSubtitle(localized, isUserAction = false) + } + + // Last resort when the scan fails outright and no AI fallback is active: behave like + // the classic non-AI flow and pick the best release-name-scored candidate. A current + // selection survives only if it's already in the target language — a fallback-language + // stand-in (e.g. embedded English picked while the addon list was still loading) must + // not outlive a failed scan. Sync is unverified, so it is deliberately not remembered + // in the match cache. + fun selectLastResort() { + if (_uiState.value.isAiTranslating) return + // AI translation first (when the feature can run) — verified timing beats any + // unverified addon pick, independent of the auto-select setting. + if (tryAiFallbackAfterScan()) return + val current = _uiState.value.selectedSubtitle + if (current != null && normalizeLanguage(current.lang) == targetLang) return + candidates.firstOrNull()?.let { + selectServedLocally(it) + showMatchToast("Selected ${it.label} (sync unverified)") + } + } + + if (loaded.isEmpty()) { + endMatch() + restoreSubtitle(previousSubtitle) + noMatch(null) + selectLastResort() + return@launch + } + + val scored = when { + builtInReference != null -> + scoreAgainstBuiltIn(loaded, builtInReference, sourceLabel, previousSubtitle) + // AI translation is already on screen as the fallback — the hearing path is too + // unreliable to risk replacing it, so just stay on AI. + _uiState.value.isAiTranslating -> null + // The hearing reference streams audio to the Gemini Live API — it needs the AI + // feature on, a key, and the Gemini model (a Groq key can't open that connection). + !aiSubtitleEnabled || aiApiKey.isBlank() || + aiModel != SubtitleAiModel.GEMINI_FLASH_25 -> null + else -> scoreAgainstHearing(loaded, sourceLabel) + } + endMatch() + + val successThreshold = if (builtInReference != null) { + MATCH_SUCCESS_THRESHOLD_TIMING + } else { + MATCH_SUCCESS_THRESHOLD_HEARING + } + val best = scored?.maxByOrNull { it.second } + if (best != null && best.second >= successThreshold) { + selectServedLocally(best.first) + // Cache the original (addon) identity — the local file is per-session transient. + writeCachedMatch(best.first) + showMatchToast("Matched: ${best.first.label} · ${(best.second * 100).toInt()}% ($sourceLabel)") + } else { + // Nothing synced found (or too little dialogue) → let the caller fall back (AI translate). + restoreSubtitle(previousSubtitle) + noMatch(best?.second) + selectLastResort() + } + } + } + + /** Whitespace/tag-insensitive form for comparing renderer cue text against parsed file text. */ + private fun normalizeCueTextForCompare(text: String): String = + text.replace(Regex("<[^>]*>"), " ").replace(Regex("\\s+"), " ").trim() + + /** Reference = a built-in track's cue timing (any language). Returns null if too little dialogue. */ + private suspend fun scoreAgainstBuiltIn( + loaded: List>>, + referenceSub: Subtitle, + sourceLabel: String, + previousSubtitle: Subtitle? + ): List>? { + synchronized(referenceIntervals) { referenceIntervals.clear() } + refIntervalStart = -1L + // If AI translation is already showing (it has the source/English track selected under the + // hood), read that track's cues without switching the displayed subtitle — keeps Hebrew on + // screen. Otherwise select the reference track so ExoPlayer renders/buffers its cues. + val aiAlreadyShowingSource = _uiState.value.isAiTranslating && aiSourceSubtitle?.id == referenceSub.id + if (!aiAlreadyShowingSource) { + hasManualSubtitleSelection = true + // Re-resolve the reference's track indices from the CURRENT list: a MediaItem rebuild + // (e.g. the previous scan's pick) reassigns embedded group indices, and a stale + // override target silently keeps the wrong track rendering. + val refResolved = _uiState.value.subtitles.firstOrNull { + it.isEmbedded && it.id == referenceSub.id && + it.groupIndex != null && it.trackIndex != null + } ?: referenceSub + _uiState.value = _uiState.value.copy( + selectedSubtitle = refResolved, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + // The reference-track selection propagates asynchronously (compose → track selector → + // decoder flush), so reading the cue buffer too early returns the *previous* track's + // cues — a previously-displayed candidate would then score a perfect match against + // itself. Give the switch a moment to settle before trusting the buffer. + delay(MATCH_TRACK_SWITCH_SETTLE_MS) + + // Unified reference collection: every tick, read the track's already-buffered (upcoming) + // cues — they land in ExoPlayer's buffer many seconds before they're spoken, so the match + // can complete before the dialogue even happens — and merge them with cues observed + // rendering in real time. Stops as soon as enough evidence accumulates. + val candMin = loaded.minOf { it.second.firstOrNull()?.startMs ?: Long.MAX_VALUE } + val candMax = loaded.maxOf { it.second.lastOrNull()?.endMs ?: 0L } + // Normalized cue texts of the candidate that was on screen when the scan started — used + // to detect stale buffer reads that still mirror that track instead of the reference. + val previousTexts: Set? = previousSubtitle?.let { prev -> + loaded.firstOrNull { it.first.id == prev.id }?.second + ?.mapTo(HashSet()) { normalizeCueTextForCompare(it.text) } + } + matchPreviousTexts = previousTexts + collectingReference = true + var refs: List> = emptyList() + var bufferedCount = 0 + var lastRefCount = 0 + var lastProgressElapsed = 0L + val startedAt = System.currentTimeMillis() + while (true) { + // Sanity-filter buffered timestamps: they must land within the candidate subtitles' + // time range, otherwise the stream-offset correction failed — ignore those. + val bufferedRaw = runCatching { + bufferedReferenceIntervalsProvider?.invoke(MATCH_BUFFERED_REF_INTERVALS) + }.getOrNull().orEmpty().filter { it.first in candMin..candMax } + // Self-match guard: verify by TEXT which track the buffer belongs to. If the buffered + // cue texts match the previously-displayed candidate's own lines, the reference track + // switch hasn't landed and we'd be scoring the candidate against itself. Timing-based + // detection is unusable here: subs cut from the same master share cue timings across + // languages (a constant-delta check froze scans by discarding legit references). + val buffered = run { + if (previousTexts.isNullOrEmpty() || bufferedRaw.isEmpty()) return@run bufferedRaw + val sampled = runCatching { bufferedCueTextsProvider?.invoke(8) } + .getOrNull().orEmpty() + .map { normalizeCueTextForCompare(it) } + .filter { it.isNotEmpty() } + if (sampled.isEmpty()) return@run bufferedRaw + val hits = sampled.count { it in previousTexts } + if (hits * 2 >= sampled.size) emptyList() else bufferedRaw + } + val realtime = synchronized(referenceIntervals) { referenceIntervals.toList() } + // Realtime intervals that overlap a buffered one describe the same cue — drop them. + val merged = buffered.toMutableList() + for (r in realtime) { + if (buffered.none { b -> r.first < b.second && b.first < r.second }) merged.add(r) + } + refs = merged.sortedBy { it.first } + bufferedCount = buffered.size + + val elapsed = System.currentTimeMillis() - startedAt + if (refs.size > lastRefCount) { + lastRefCount = refs.size + lastProgressElapsed = elapsed + } + // While paused no realtime evidence can arrive — freeze the give-up clock so a scan + // with partial evidence isn't concluded just because the user paused. (A scan with + // full evidence still exits via the target/early-accept breaks above.) + if (!lastIsPlaying) lastProgressElapsed = elapsed + // Interval count alone is not evidence: cues clustered in one moment (a recap, a + // titles sequence) can agree with an overall-drifting sub. Only accept once the refs + // also cover a minimum stretch of the video. + val span = if (refs.size >= 2) refs.last().second - refs.first().first else 0L + val spanOk = span >= MATCH_MIN_REF_SPAN_MS + if (spanOk && refs.size >= MATCH_TARGET_REF_INTERVALS) break + // Early accept: scoring is free, so score incrementally — once some candidate is + // already clearly in sync there's no need to keep collecting up to the target. + val bestNow = if (refs.size >= MATCH_EARLY_ACCEPT_REFS) { + loaded.maxOf { (_, cues) -> SubtitleSyncMatcher.scoreByTiming(cues, refs) } + } else { + 0.0 + } + if (spanOk && refs.size >= MATCH_EARLY_ACCEPT_REFS && bestNow >= MATCH_EARLY_ACCEPT_SCORE) break + // Fast accept: a decisively-synced candidate doesn't need the full span — the + // cluster false-positive it protects against packs refs into a few seconds, which + // half the span already rules out. Saves ~15s of realtime collection per scan. + if (span >= MATCH_FAST_ACCEPT_SPAN_MS && refs.size >= MATCH_FAST_ACCEPT_REFS && + bestNow >= MATCH_FAST_ACCEPT_SCORE + ) break + // Give-up timers: a long one while waiting for speech to exist at all (openings can + // run minutes without dialogue), and a progress-anchored one afterwards — a silent + // scene mid-scan pauses the clock instead of draining it. Absolute ceiling on top. + val deadline = if (lastRefCount == 0) { + MATCH_NO_SPEECH_CAP_MS + } else { + minOf(lastProgressElapsed + MATCH_HARD_CAP_MS, MATCH_TOTAL_CAP_MS) + } + if (elapsed >= deadline) break + updateMatchStatus( + if (refs.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" + else "Searching for a match ($sourceLabel) — (${refs.size})" + ) + delay(300) + } + collectingReference = false + matchPreviousTexts = null + // Inconclusive after the hard cap (too few refs, or all clustered in one moment) — a pick + // would be a guess; return null so the caller stays on the AI-translation fallback. + // Floor matches the fast-accept span so a fast-accepted result isn't discarded here. + if (refs.size < MATCH_MIN_REF_INTERVALS || + refs.last().second - refs.first().first < MATCH_FAST_ACCEPT_SPAN_MS + ) return null + val srcLabel = when { + bufferedCount == refs.size -> "buffer" + bufferedCount == 0 -> "realtime" + else -> "buffer+realtime" + } + + loaded.firstOrNull()?.let { (_, cues) -> + // Show the candidate cues NEAREST the first reference window — file-start cues say + // nothing about alignment at the scan position. + val firstRef = refs.firstOrNull()?.first ?: 0L + val nearIdx = cues.indexOfFirst { it.endMs >= firstRef }.coerceAtLeast(0) + val near = cues.subList(nearIdx, minOf(nearIdx + 3, cues.size)) + android.util.Log.i( + "SubMatch", + "align src=$srcLabel buffered=$bufferedCount total=${refs.size} " + + "refs=${refs.take(3)} candNear=${near.map { it.startMs to it.endMs }} " + + "candRange=${cues.firstOrNull()?.startMs}..${cues.lastOrNull()?.endMs}" + ) + } + + return loaded.map { (sub, cues) -> + val s = SubtitleSyncMatcher.scoreByTiming(cues, refs) + android.util.Log.i( + "SubMatch", + "[builtin] candidate label=\"${sub.label}\" provider=${sub.provider} cues=${cues.size} " + + "refIntervals=${refs.size} src=$srcLabel score=${"%.2f".format(s)}" + ) + sub to s + } + } + + /** Reference = AI hearing transcription (fallback when there's no built-in English track). */ + private suspend fun scoreAgainstHearing( + loaded: List>>, + sourceLabel: String + ): List>? { + startMatchListening() + val samples = mutableListOf() + val collector = viewModelScope.launch { + geminiLiveService.translatedText + .filterNotNull() + .distinctUntilChanged() + .collect { line -> + val t = line.trim() + if (t.length >= 2) samples.add(SubtitleSyncMatcher.SpokenSample(t, lastKnownPositionMs)) + } + } + val startedAt = System.currentTimeMillis() + var lastSampleCount = 0 + var lastProgressElapsed = 0L + while (samples.size < MATCH_MAX_SAMPLES) { + val elapsed = System.currentTimeMillis() - startedAt + if (samples.size > lastSampleCount) { + lastSampleCount = samples.size + lastProgressElapsed = elapsed + } + // Paused playback produces no audio — freeze the give-up clock (as in the built-in path). + if (!lastIsPlaying) lastProgressElapsed = elapsed + // Same timer scheme as the built-in path: wait long for speech to exist, then a + // progress-anchored budget so mid-scan silence doesn't drain it. Absolute ceiling. + val deadline = if (lastSampleCount == 0) { + MATCH_NO_SPEECH_CAP_MS + } else { + minOf(lastProgressElapsed + MATCH_HARD_CAP_MS, MATCH_TOTAL_CAP_MS) + } + if (elapsed >= deadline) break + // Connection failed (bad key, network) → no samples will ever arrive; bail out now + // instead of spinning until the hard cap. + if (geminiLiveService.state.value == GeminiLiveState.ERROR) { + android.util.Log.w("SubMatch", "hearing aborted: Gemini Live error=${geminiLiveService.errorMessage.value}") + break + } + updateMatchStatus( + if (samples.isEmpty()) "Searching for a match ($sourceLabel) — waiting for speech…" + else "Searching for a match ($sourceLabel) — scanning subtitles… (${samples.size})" + ) + delay(300) + } + collector.cancel() + if (samples.size < MATCH_MIN_SAMPLES) return null + + return loaded.map { (sub, cues) -> + val s = SubtitleSyncMatcher.score( + cues, samples, MATCH_LATENCY_MS, MATCH_TOLERANCE_MS, MATCH_MIN_SIMILARITY + ) + android.util.Log.i( + "SubMatch", + "[hearing] candidate label=\"${sub.label}\" provider=${sub.provider} cues=${cues.size} " + + "samples=${samples.size} score=${"%.2f".format(s)}" + ) + sub to s + } + } + + private fun beginMatch(status: String) { + _uiState.value = _uiState.value.copy(isFindingBestMatch = true, matchStatusText = status) + } + + private fun updateMatchStatus(status: String) { + if (_uiState.value.matchStatusText != status) { + _uiState.value = _uiState.value.copy(matchStatusText = status) + } + } + + private fun startMatchListening() { + hasManualSubtitleSelection = true + subtitleSelectionJob?.cancel() + translationManager.isEnabled = false + targetSubtitleLangCode.takeIf { it.isNotBlank() }?.let { geminiLiveService.targetLanguageCode = it } + geminiLiveService.connect() + _uiState.value = _uiState.value.copy( + isLiveAudioTranslating = true, + selectedSubtitle = null, + isAiTranslating = false, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + private fun endMatch() { + geminiLiveService.disconnect() + _uiState.value = _uiState.value.copy( + isFindingBestMatch = false, + isLiveAudioTranslating = false, + matchStatusText = "" + ) + } + + private fun restoreSubtitle(subtitle: Subtitle?) { + _uiState.value = _uiState.value.copy( + selectedSubtitle = subtitle, + subtitleSelectionNonce = _uiState.value.subtitleSelectionNonce + 1 + ) + } + + // ── Per-stream match cache ────────────────────────────────────────────────── + + private data class CachedSubMatch(val key: String, val provider: String, val id: String) + + /** + * Identity of the currently playing file, stable across sessions. Torrent infoHash+fileIdx is + * best; videoHash/filename+size next; the raw URL last (debrid links change per resolve, so a + * URL key may simply never hit again — harmless). + */ + private fun currentStreamCacheKey(): String? { + val stream = _uiState.value.selectedStream + if (stream != null) { + val infoHash = stream.infoHash + if (!infoHash.isNullOrBlank()) return "ih:$infoHash:${stream.fileIdx ?: -1}" + val hints = stream.behaviorHints + val videoHash = hints?.videoHash + if (!videoHash.isNullOrBlank()) return "vh:$videoHash" + val filename = hints?.filename + if (!filename.isNullOrBlank()) { + return "fn:$filename:${hints.videoSize ?: stream.sizeBytes ?: -1}" + } + } + val url = stream?.url?.takeIf { it.isNotBlank() } ?: _uiState.value.selectedStreamUrl + return url?.takeIf { it.isNotBlank() }?.let { "url:${it.hashCode()}" } + } + + private suspend fun loadMatchCache(): MutableList { + val json = context.settingsDataStore.data.first()[subtitleMatchCacheKey] + if (json.isNullOrBlank()) return mutableListOf() + val type = TypeToken.getParameterized(MutableList::class.java, CachedSubMatch::class.java).type + return runCatching { gson.fromJson>(json, type) } + .getOrNull() ?: mutableListOf() + } + + private suspend fun readCachedMatch(): CachedSubMatch? { + val key = currentStreamCacheKey() ?: return null + return loadMatchCache().lastOrNull { it.key == key } + } + + /** Remember [subtitle] as the match for the current stream (null = forget the entry). */ + private fun writeCachedMatch(subtitle: Subtitle?) { + val key = currentStreamCacheKey() ?: return + viewModelScope.launch { + runCatching { + val cache = loadMatchCache() + val removed = cache.removeAll { it.key == key } + if (subtitle == null && !removed) return@runCatching + if (subtitle != null) cache.add(CachedSubMatch(key, subtitle.provider, subtitle.id)) + while (cache.size > MATCH_CACHE_MAX_ENTRIES) cache.removeAt(0) + context.settingsDataStore.edit { it[subtitleMatchCacheKey] = gson.toJson(cache) } + }.onFailure { Log.w("SubMatch", "match cache write failed: ${it.message}") } + } + } + + /** + * A manual track pick overrides what the matcher remembered for this stream: remember the + * user's choice if it's an addon sub in the AI target language, otherwise drop the entry so + * the stale match can't fight the user on the next playback. + */ + private fun updateMatchCacheForManualPick(subtitle: Subtitle) { + val target = targetSubtitleLangCode + val cacheable = target.isNotBlank() && !subtitle.isEmbedded && !subtitle.isBitmap && + subtitle.url.isNotBlank() && normalizeLanguage(subtitle.lang) == target + writeCachedMatch(if (cacheable) subtitle else null) + } + + /** + * Writes downloaded subtitle text to a small local cache file and returns a copy of [sub] + * pointing at it (file:// URI). ExoPlayer then side-loads the matched subtitle instantly + * during the MediaItem rebuild instead of re-downloading it from a possibly slow/flaky addon + * server — which is what used to leave the player stuck buffering after a match. Also + * normalizes the content (UTF-8, gunzipped) as a side effect. + */ + private fun localizeSubtitle(sub: Subtitle, raw: String): Subtitle { + return runCatching { + val dir = java.io.File(context.cacheDir, "matched_subs").apply { mkdirs() } + // Keep the directory bounded; files are ~100KB and keyed stably per subtitle. + dir.listFiles()?.sortedBy { it.lastModified() }?.dropLast(39)?.forEach { it.delete() } + val ext = if (raw.trimStart().startsWith("WEBVTT")) "vtt" else "srt" + val file = java.io.File(dir, "${("${sub.provider}|${sub.id}").hashCode().toUInt()}.$ext") + file.writeText(raw) + sub.copy(url = file.toURI().toString()) + }.getOrDefault(sub) + } + + private fun showMatchToast(message: String) { + _uiState.value = _uiState.value.copy(matchToast = message) + } + + fun dismissMatchToast() { + _uiState.value = _uiState.value.copy(matchToast = null) + } + fun disableSubtitles() { hasManualSubtitleSelection = true subtitleSelectionJob?.cancel() @@ -3092,6 +4070,38 @@ class PlayerViewModel @Inject constructor( companion object { private const val TAG = "PlayerViewModel" + // ── "Find best match" tuning ────────────────────────────────────────── + private const val MATCH_HARD_CAP_MS = 90_000L // give up this long after the FIRST cue/sample appeared + private const val MATCH_NO_SPEECH_CAP_MS = 240_000L // how long to wait for speech to exist at all + private const val MATCH_TOTAL_CAP_MS = 600_000L // absolute ceiling on a single scan + private const val MATCH_MAX_REF_INTERVAL_MS = 20_000L // longer "cues" are seek artifacts + private const val MATCH_MAX_SAMPLES = 12 // stop early once we have this many lines + private const val MATCH_MIN_REF_INTERVALS = 2 // built-in mode: min reference cues to decide (1 is too noisy) + private const val MATCH_TARGET_REF_INTERVALS = 8 // built-in mode: stop collecting once we have this many + private const val MATCH_EARLY_ACCEPT_REFS = 4 // min refs before the incremental early-accept check + private const val MATCH_EARLY_ACCEPT_SCORE = 0.8 // a candidate this well-synced ends collection early + private const val MATCH_FAST_ACCEPT_SPAN_MS = 15_000L // decisive candidates accept at half the span + private const val MATCH_FAST_ACCEPT_REFS = 5 // …with at least this many refs + private const val MATCH_FAST_ACCEPT_SCORE = 0.85 // …scoring at least this + private const val MATCH_MIN_REF_SPAN_MS = 30_000L // refs must cover this much video before any accept + private const val MATCH_TRACK_SWITCH_SETTLE_MS = 1_500L // let the reference-track switch reach the renderer + private const val MATCH_BUFFERED_REF_INTERVALS = 12 // max upcoming cues to read from the buffer per poll + private const val MATCH_EMBEDDED_WAIT_MS = 8_000L // grace period for async embedded tracks + private const val MATCH_SOURCES_WAIT_MS = 15_000L // max continuous-playback wait for embedded + addon subs + private const val MATCH_PLAYBACK_WAIT_CAP_MS = 90_000L // give up if playback never starts at all + private const val MATCH_MIN_SAMPLES = 4 // need at least this many to decide + private const val MATCH_LATENCY_MS = 800L // AI hearing lag: audio time ≈ arrival − this + private const val MATCH_TOLERANCE_MS = 1_500L // cue search window around the audio time + private const val MATCH_MIN_SIMILARITY = 0.35 // word-overlap needed to count a cue as a hit + // Separate accept thresholds per reference type: calibrated against user-verified subs — + // a confirmed-synced sub scored 0.71, a confirmed-offset one 0.65 (union-coverage + // scoring, 12 refs). Below the bar, staying on AI translation beats a visibly-off sub. + // Hearing hit-ratios run much lower, so they keep the permissive bar. + private const val MATCH_SUCCESS_THRESHOLD_TIMING = 0.70 + private const val MATCH_SUCCESS_THRESHOLD_HEARING = 0.30 + private const val MATCH_MAX_CANDIDATES = 10 // cap downloaded candidates (best release-name matches first) + private const val MATCH_CACHE_MAX_ENTRIES = 50 // per-stream remembered matches (oldest evicted) + /** Known debrid service CDN domains. Reachability checks are skipped for these. */ private val DEBRID_CDN_DOMAINS = setOf( // Real-Debrid diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt new file mode 100644 index 000000000..ef210f3f4 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleSyncMatcher.kt @@ -0,0 +1,209 @@ +package com.arflix.tv.ui.screens.player + +import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import java.util.zip.GZIPInputStream +import java.util.concurrent.TimeUnit + +/** + * "Find best match": scores candidate subtitle tracks by how well their on-screen cue at a given + * moment matches what the AI hearing transcribed being spoken at that same moment. A well-synced + * subtitle shows the right words at the right time; an out-of-sync one does not. The comparison + * is language-agnostic word overlap (normalization additionally strips Hebrew niqqud and bidi + * marks, which is a no-op for other languages); callers pass candidates in the user's preferred + * subtitle language. + */ +object SubtitleSyncMatcher { + + private const val TAG = "SubMatch" + + data class TimedCue(val startMs: Long, val endMs: Long, val text: String) + + /** One collected AI-hearing sample: the spoken line and the media position when it arrived. */ + data class SpokenSample(val text: String, val positionMs: Long) + + private val client = OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(15, TimeUnit.SECONDS) + .build() + + /** Downloads a subtitle and returns its decoded (UTF-8, gunzipped) text, or null on failure. */ + suspend fun loadRaw(url: String): String? = withContext(Dispatchers.IO) { + runCatching { + val request = Request.Builder().url(url).build() + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) return@use null + val body = response.body ?: return@use null + val raw = body.bytes() + if (looksGzipped(url, raw)) { + GZIPInputStream(raw.inputStream()).bufferedReader(Charsets.UTF_8).readText() + } else { + String(raw, Charsets.UTF_8) + } + } + }.onFailure { error -> + Log.w(TAG, "loadRaw failed url=$url err=${error.message}") + }.getOrNull() + } + + suspend fun loadCues(url: String): List = loadRaw(url)?.let { parseCues(it) } ?: emptyList() + + /** + * Score a candidate's cues against the collected spoken samples. For each sample we look at + * the audio's media time (arrival position minus the AI latency), find cues within a tolerance + * window, and count a hit when the best word-overlap clears [minSimilarity]. Returns the hit + * ratio in 0..1. + */ + fun score( + cues: List, + samples: List, + latencyMs: Long, + toleranceMs: Long, + minSimilarity: Double + ): Double { + if (cues.isEmpty() || samples.isEmpty()) return 0.0 + var hits = 0 + for (sample in samples) { + val audioTime = sample.positionMs - latencyMs + val best = cues.asSequence() + .filter { it.endMs >= audioTime - toleranceMs && it.startMs <= audioTime + toleranceMs } + .maxOfOrNull { similarity(sample.text, it.text) } ?: 0.0 + if (best >= minSimilarity) hits++ + } + return hits.toDouble() / samples.size + } + + /** Reference windows longer than this are merged back-to-back cues, not a single cue. */ + private const val SINGLE_CUE_MAX_MS = 7_000L + + /** + * Score a candidate against a synced reference (a built-in subtitle's cue-visible intervals) + * purely by timing, averaging a per-window overlap fraction. + * + * Two regimes per reference window: + * - Normal windows (a single cue, ≤ [SINGLE_CUE_MAX_MS]): best overlap by any *one* candidate + * cue. This is what discriminates offsets — in dialogue-dense scenes a shifted sub still has + * *chains* of cues touching any window, but no single cue aligns with it. + * - Long windows (realtime collection merges back-to-back cues): coverage by the union of the + * candidate's cues, since even a perfectly synced sub needs several cues to span them. + * + * Language-agnostic (no translation needed). + */ + fun scoreByTiming(cues: List, referenceIntervals: List>): Double { + if (cues.isEmpty() || referenceIntervals.isEmpty()) return 0.0 + val sorted = cues.sortedBy { it.startMs } + var sum = 0.0 + for ((rs, re) in referenceIntervals) { + val refDur = (re - rs).coerceAtLeast(1L) + val fraction = if (refDur <= SINGLE_CUE_MAX_MS) { + sorted.maxOfOrNull { c -> + (minOf(re, c.endMs) - maxOf(rs, c.startMs)).toDouble() / refDur + } ?: 0.0 + } else { + var covered = 0L + var cursor = rs + for (c in sorted) { + if (c.endMs <= cursor) continue + if (c.startMs >= re) break + val s = maxOf(c.startMs, cursor) + val e = minOf(c.endMs, re) + if (e > s) { + covered += e - s + cursor = e + } + } + covered.toDouble() / refDur + } + sum += fraction.coerceIn(0.0, 1.0) + } + return sum / referenceIntervals.size + } + + // ── Similarity ──────────────────────────────────────────────────────────── + + /** Overlap coefficient of normalized word sets — tolerant of different wording/length. */ + fun similarity(a: String, b: String): Double { + val setA = tokens(a) + val setB = tokens(b) + if (setA.isEmpty() || setB.isEmpty()) return 0.0 + val intersection = setA.count { it in setB } + return intersection.toDouble() / minOf(setA.size, setB.size) + } + + private fun tokens(text: String): Set = + normalize(text).split(' ').filter { it.length >= 2 }.toSet() + + private fun normalize(text: String): String { + val sb = StringBuilder(text.length) + for (ch in text) { + when { + // Strip Hebrew niqqud/cantillation and bidi control marks. + ch in '֑'..'ׇ' -> {} + ch == '‎' || ch == '‏' || ch == '‪' || ch == '‫' || + ch == '‬' || ch == '⁦' || ch == '⁧' || ch == '⁩' -> {} + ch.isLetterOrDigit() -> sb.append(ch.lowercaseChar()) + else -> sb.append(' ') + } + } + return sb.toString().replace(Regex("\\s+"), " ").trim() + } + + fun cueTextAt(cues: List, timeMs: Long): String? = + cues.firstOrNull { timeMs in it.startMs..it.endMs }?.text + + // ── Parsing (SRT + WEBVTT) ────────────────────────────────────────────────── + + private fun looksGzipped(url: String, bytes: ByteArray): Boolean = + url.substringBefore('?').endsWith(".gz", ignoreCase = true) || + (bytes.size > 2 && bytes[0] == 0x1f.toByte() && bytes[1] == 0x8b.toByte()) + + private val TIME_LINE = Regex( + """(\d{1,2}:\d{2}:\d{2}[.,]\d{1,3}|\d{1,2}:\d{2}[.,]\d{1,3})\s*-->\s*(\d{1,2}:\d{2}:\d{2}[.,]\d{1,3}|\d{1,2}:\d{2}[.,]\d{1,3})""" + ) + private val TAG_STRIP = Regex("<[^>]*>") + + fun parseCues(content: String): List { + val normalized = content.replace("\r\n", "\n").replace('\r', '\n') + val blocks = normalized.split(Regex("\n\\s*\n")) + val cues = ArrayList() + for (block in blocks) { + val lines = block.split('\n').map { it.trim() }.filter { it.isNotEmpty() } + if (lines.isEmpty()) continue + val timeLineIndex = lines.indexOfFirst { TIME_LINE.containsMatchIn(it) } + if (timeLineIndex < 0) continue + val match = TIME_LINE.find(lines[timeLineIndex]) ?: continue + val start = parseTimestamp(match.groupValues[1]) ?: continue + val end = parseTimestamp(match.groupValues[2]) ?: continue + val text = lines.drop(timeLineIndex + 1) + .joinToString(" ") + .replace(TAG_STRIP, "") + .trim() + if (text.isNotEmpty() && end > start) cues.add(TimedCue(start, end, text)) + } + return cues + } + + private fun parseTimestamp(value: String): Long? { + val cleaned = value.replace(',', '.') + val parts = cleaned.split(':') + return runCatching { + when (parts.size) { + 3 -> { + val h = parts[0].toLong() + val m = parts[1].toLong() + val s = parts[2].toDouble() + (h * 3600 + m * 60) * 1000 + (s * 1000).toLong() + } + 2 -> { + val m = parts[0].toLong() + val s = parts[1].toDouble() + (m * 60) * 1000 + (s * 1000).toLong() + } + else -> null + } + }.getOrNull() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt index aaed83579..913f88066 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/SubtitleTranslationManager.kt @@ -140,6 +140,10 @@ class SubtitleTranslationManager( } suspend fun preTranslateWindow(texts: List) { + // The lookahead triggers fire whenever a text track renders cues — including tracks + // selected under the hood by "find best match" with AI translation off. Never spend + // API requests unless translation is actually active. + if (!isEnabled) return val uncached = texts.filter { !cache.containsKey(it) && !inFlight.containsKey(it) } if (uncached.isEmpty()) return uncached.chunked(40).forEach { chunk -> diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 1c56df70b..6a32a1401 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -216,7 +216,7 @@ private val tvGeneralSectionIds = setOf( private fun tvGeneralRowsForSection(section: String): List { return when (section) { "language" -> listOf(0, 3) - "subtitles" -> listOf(1, 2, 4, 5, 6, 7, 8, 9) + "subtitles" -> listOf(1, 2, 38, 4, 5, 6, 7, 8, 9) "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) "playback" -> listOf(10, 11, 12, 13, 14, 37, 34, 16, 15, 27) "appearance" -> listOf(17, 18, 20, 21, 24, 23, 22, 36) @@ -903,6 +903,7 @@ fun SettingsScreen( 28 -> viewModel.setSubtitleAiEnabled(!uiState.subtitleAiEnabled) 29 -> showAiModelDialog = true 30 -> viewModel.setSubtitleAiAutoSelect(!uiState.subtitleAiAutoSelect) + 38 -> viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() @@ -1330,12 +1331,14 @@ fun SettingsScreen( onQualityFiltersClick = { showQualityFiltersModal = true }, subtitleAiEnabled = uiState.subtitleAiEnabled, subtitleAiAutoSelect = uiState.subtitleAiAutoSelect, + subtitleAiFindBestMatch = uiState.subtitleAiFindBestMatch, subtitleAiApiKey = uiState.subtitleAiApiKey, subtitleAiModel = uiState.subtitleAiModel, subtitleRemoveHearingImpaired = uiState.subtitleRemoveHearingImpaired, onSubtitleAiEnabledToggle = { viewModel.setSubtitleAiEnabled(it) }, onSubtitleAiModelClick = { showAiModelDialog = true }, onSubtitleAiAutoSelectToggle = { viewModel.setSubtitleAiAutoSelect(it) }, + onSubtitleAiFindBestMatchToggle = { viewModel.setSubtitleAiFindBestMatch(it) }, onSubtitleRemoveHearingImpairedToggle = { viewModel.setSubtitleRemoveHearingImpaired(it) }, onSubtitleAiApiKeyClick = { showAiApiKeyDialog = true }, onSubtitleAiQrClick = { viewModel.startAiKeyServer() }, @@ -3420,6 +3423,15 @@ private fun MobileSettingsMainPage( isFocused = false, onClick = openSecondarySubtitlePicker ) + // AI-independent: the timing-based match scan needs no API key. + MobileSettingsRow( + icon = Icons.Default.Subtitles, + title = stringResource(R.string.ai_find_best_match_title), + subtitle = stringResource(R.string.ai_find_best_match_desc), + value = if (uiState.subtitleAiFindBestMatch) "On" else "Off", + isFocused = false, + onClick = { viewModel.setSubtitleAiFindBestMatch(!uiState.subtitleAiFindBestMatch) } + ) MobileSettingsRow( icon = Icons.Default.Subtitles, title = stringResource(R.string.filter_subtitles), @@ -4738,12 +4750,14 @@ private fun TvGeneralSettingsRows( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, + subtitleAiFindBestMatch: Boolean = false, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, onSubtitleAiEnabledToggle: (Boolean) -> Unit = {}, onSubtitleAiModelClick: () -> Unit = {}, onSubtitleAiAutoSelectToggle: (Boolean) -> Unit = {}, + onSubtitleAiFindBestMatchToggle: (Boolean) -> Unit = {}, onSubtitleRemoveHearingImpairedToggle: (Boolean) -> Unit = {}, onSubtitleAiApiKeyClick: () -> Unit = {}, onSubtitleAiQrClick: () -> Unit = {}, @@ -4850,6 +4864,8 @@ private fun TvGeneralSettingsRows( modifier = Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f) ) 30 -> SettingsToggleRow(stringResource(R.string.ai_auto_select_title), stringResource(R.string.ai_auto_select_desc), subtitleAiAutoSelect, focusedIndex == localIndex, onSubtitleAiAutoSelectToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) + // AI-independent: the timing-based match scan needs no API key. + 38 -> SettingsToggleRow(stringResource(R.string.ai_find_best_match_title), stringResource(R.string.ai_find_best_match_desc), subtitleAiFindBestMatch, focusedIndex == localIndex, onSubtitleAiFindBestMatchToggle, Modifier.settingsFocusSlot(localIndex)) 31 -> SettingsToggleRow(stringResource(R.string.ai_remove_hi_title), stringResource(R.string.ai_remove_hi_desc), subtitleRemoveHearingImpaired, focusedIndex == localIndex, onSubtitleRemoveHearingImpairedToggle, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 32 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.ai_api_key_title), stringResource(R.string.ai_api_key_desc), maskAiApiKey(subtitleAiApiKey, stringResource(R.string.ai_key_not_set)), focusedIndex == localIndex, onSubtitleAiApiKeyClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) 33 -> SettingsRow(Icons.Default.QrCode, stringResource(R.string.ai_scan_qr_title), stringResource(R.string.ai_scan_qr_desc), "", focusedIndex == localIndex, onSubtitleAiQrClick, Modifier.settingsFocusSlot(localIndex).alpha(if (subtitleAiEnabled) 1f else 0.4f)) @@ -4923,12 +4939,14 @@ private fun GeneralSettings( onQualityFiltersClick: () -> Unit = {}, subtitleAiEnabled: Boolean = false, subtitleAiAutoSelect: Boolean = false, + subtitleAiFindBestMatch: Boolean = false, subtitleAiApiKey: String = "", subtitleAiModel: com.arflix.tv.ui.screens.player.SubtitleAiModel = com.arflix.tv.ui.screens.player.SubtitleAiModel.GROQ_LLAMA_70B, subtitleRemoveHearingImpaired: Boolean = true, onSubtitleAiEnabledToggle: (Boolean) -> Unit = {}, onSubtitleAiModelClick: () -> Unit = {}, onSubtitleAiAutoSelectToggle: (Boolean) -> Unit = {}, + onSubtitleAiFindBestMatchToggle: (Boolean) -> Unit = {}, onSubtitleRemoveHearingImpairedToggle: (Boolean) -> Unit = {}, onSubtitleAiApiKeyClick: () -> Unit = {}, onSubtitleAiQrClick: () -> Unit = {}, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 71afebbd8..d6dab3872 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -197,6 +197,7 @@ data class SettingsUiState( // AI Subtitles val subtitleAiEnabled: Boolean = false, val subtitleAiAutoSelect: Boolean = false, + val subtitleAiFindBestMatch: Boolean = false, val subtitleAiApiKey: String = "", val subtitleAiModel: SubtitleAiModel = SubtitleAiModel.GROQ_LLAMA_70B, val subtitleRemoveHearingImpaired: Boolean = true, @@ -286,6 +287,7 @@ class SettingsViewModel @Inject constructor( // Global (non-profile-scoped) AI subtitle settings — device-wide, not per-profile private val subtitleAiEnabledKey = booleanPreferencesKey("subtitle_ai_enabled") private val subtitleAiAutoSelectKey = booleanPreferencesKey("subtitle_ai_auto_select") + private val subtitleAiFindBestMatchKey = booleanPreferencesKey("subtitle_ai_find_best_match") private val subtitleAiApiKeyKey = stringPreferencesKey("subtitle_ai_api_key") private val subtitleAiModelKey = stringPreferencesKey("subtitle_ai_model") private val subtitleRemoveHearingImpairedKey = booleanPreferencesKey("subtitle_remove_hearing_impaired") @@ -489,6 +491,7 @@ class SettingsViewModel @Inject constructor( val subtitleAiEnabled = prefs[subtitleAiEnabledKey] ?: false val subtitleAiAutoSelect = prefs[subtitleAiAutoSelectKey] ?: false + val subtitleAiFindBestMatch = prefs[subtitleAiFindBestMatchKey] ?: false val subtitleAiApiKey = prefs[subtitleAiApiKeyKey] ?: "" val subtitleAiModel = runCatching { SubtitleAiModel.valueOf(prefs[subtitleAiModelKey] ?: SubtitleAiModel.GROQ_LLAMA_70B.name) @@ -558,6 +561,7 @@ class SettingsViewModel @Inject constructor( qualityFilterPresetLabel = detectQualityFilterPreset(qualityFilters).label, subtitleAiEnabled = subtitleAiEnabled, subtitleAiAutoSelect = subtitleAiAutoSelect, + subtitleAiFindBestMatch = subtitleAiFindBestMatch, subtitleAiApiKey = subtitleAiApiKey, subtitleAiModel = subtitleAiModel, subtitleRemoveHearingImpaired = subtitleRemoveHearingImpaired, @@ -1298,6 +1302,14 @@ class SettingsViewModel @Inject constructor( } } + fun setSubtitleAiFindBestMatch(enabled: Boolean) { + viewModelScope.launch { + context.settingsDataStore.edit { it[subtitleAiFindBestMatchKey] = enabled } + _uiState.value = _uiState.value.copy(subtitleAiFindBestMatch = enabled) + syncLocalStateToCloud(silent = true) + } + } + fun setSubtitleRemoveHearingImpaired(enabled: Boolean) { viewModelScope.launch { context.settingsDataStore.edit { it[subtitleRemoveHearingImpairedKey] = enabled } diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index 28649c613..8116ea887 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -208,6 +208,8 @@ בחר ספק ומודל לתרגום בחירה אוטומטית של תרגום בינה מלאכותית הפעל אוטומטית כאשר קיים תרגום מובנה כלשהו + חיפוש אוטומטי של הכתובית המתאימה ביותר + בחירה אוטומטית של הכתובית המסונכרנת ביותר מההרחבות שלך הסר טקסט למוגבלי שמיעה הסר סימוני [סוגריים] למוגבלי שמיעה לפני תרגום מפתח API diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index eed488ca4..cff06215d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -235,6 +235,8 @@ Select translation provider and model Auto-Select AI Translation Automatically activate AI when any built in language is available + Auto Find Best Subtitle Match + Auto-pick the best-synced subtitle from your addons Remove Hearing Impaired Text Strip [bracketed] hearing-impaired markers before translation API Key diff --git a/docs/subtitle-auto-match.md b/docs/subtitle-auto-match.md new file mode 100644 index 000000000..542a14abd --- /dev/null +++ b/docs/subtitle-auto-match.md @@ -0,0 +1,234 @@ +# Subtitle Auto-Match ("Find Best Match") — Design & Behavior Reference + +Complete reference for the subtitle auto-matching feature and its interaction with AI subtitle +translation. Written to be self-contained: hand this file to a fresh session/developer for full +context. + +**Code map** + +| Concern | File | +|---|---| +| Scan orchestration, selection flows, caches, settings gating | `app/src/main/kotlin/com/arflix/tv/ui/screens/player/PlayerViewModel.kt` (`findBestSubtitleMatch`, `runFindBestMatch`, `activateAiSubtitle`, `applyPreferredSubtitle`, `MATCH_*` constants) | +| Cue download/parse + timing/word scoring | `.../player/SubtitleSyncMatcher.kt` | +| Subtitle menu, selection application, media item rebuild, startup watchdog | `.../player/PlayerScreen.kt` | +| AI batch translation (Groq/Gemini generateContent) | `.../player/SubtitleTranslationManager.kt`, `SubtitleTranslationService.kt` | +| Live hearing (Gemini Live WS, audio → target-language text) | `.../player/GeminiLiveTranslationService.kt`, `AudioCaptureProcessor.kt` | +| Renderer hooks (buffered-cue reflection, pre-translation, cue offset) | `.../player/AiSubtitleRenderersFactory.kt` | +| Settings UI (toggle lives in **Subtitles** section, not AI) | `.../settings/SettingsScreen.kt` (TV row id **38**), `SettingsViewModel.kt` | +| Cloud backup/restore of settings | `.../data/repository/CloudSyncRepository.kt` | + +--- + +## 1. Settings and what they control + +| Setting | DataStore key | Default | Controls | +|---|---|---|---| +| Preferred subtitle language | `default_subtitle` (profile-scoped) | — | The **target language** of matching and AI translation (`targetSubtitleLangCode`, `matchLanguageName`). | +| Auto Find Best Subtitle Match | `subtitle_ai_find_best_match` | `false` | Auto-runs the scan on playback. **AI-independent** (works with no API key). Off by default — users opt in. Key name kept for backward compat. | +| AI subtitle translation (master) | `subtitle_ai_enabled` | `false` | Enables AI features: translation option in menu, AI interim during scans, hearing fallback. | +| Auto-Select AI Translation | `subtitle_ai_auto_select` | `false` | Allows AI translation to activate **automatically** (incl. as the scan's on-screen interim). | +| Model | `subtitle_ai_model` | `GROQ_LLAMA_70B` | Groq or Gemini for batch translation. **Hearing requires Gemini.** | +| API key | `subtitle_ai_api_key` (global) | — | One key used for both batch translation and Gemini Live. | + +All of the above (except profile-scoped language) sync via `CloudSyncRepository` +(`subtitleAiFindBestMatch` field in the backup JSON). + +### Behavior matrix (all combinations) + +| AI master | Auto-select AI | Find-match | On playback | +|---|---|---|---| +| any | any | **off** | Classic flow only: best release-name-scored sub in preferred language. Manual "Find Best Match" menu entry still available. | +| off | — | on | Classic pick shows immediately → **silent background scan** (subtitles hidden during it) → verified match replaces pick, else pick stays. Zero AI/API usage. | +| on | off | on | Same silent scan (user opted out of auto-AI). AI translate remains a manual menu option. | +| on | on | off | AI translation activates directly (no scan). | +| on | on | on | Scan with **AI translation as on-screen interim**; if no match, AI stays (source re-resolved to prefer embedded). | + +Menu entries (player → subtitle menu, inside the preferred-language group): +- **"Find Best Match"** (badge `Auto`) — always present when a preferred language is set (`matchLanguageName`), AI-independent. +- **"" AI translate** (badge `AI`) — only when AI available (`isAiAvailable`). +- D-pad index math: headers = `(match? 1 : 0) + (ai? 1 : 0)`, subs follow. + +--- + +## 2. The scan pipeline + +```mermaid +flowchart TD + A[Scan requested] --> B{Trigger} + B -->|auto: applyPreferredSubtitle, once per stream
latch: autoMatchAttempted| C + B -->|manual: menu entry
clears remembered cache entry, useCache=false| C + C[Wait for sources
continuous-playback clock: embedded grace 8s,
candidates wait 15s, absolute 90s] --> D{Embedded track in
preferred language?} + D -->|yes| E[Select it — muxed = guaranteed sync
toast: Matched embedded] + D -->|no| F[Candidates: external subs in pref language,
sorted by release-name score, capped at 10] + F -->|none| G[toast: no well-synced subtitle] + F --> H{Remembered match for this exact stream?
only when useCache} + H -->|hit| I[Re-download text → local file → select
toast: remembered] + H -->|miss| J[Download all candidates in parallel
keep raw text + parsed cues] + J --> K{Reference source?} + K -->|any embedded non-forced non-bitmap track,
English preferred| L[Timing scan] + K -->|none + AI on + key + Gemini| M[Hearing scan:
Gemini Live audio→text, word overlap,
threshold 0.30] + K -->|none, otherwise| N[Inconclusive] + L --> O{Best score ≥ 0.70?} + M --> O2{Best ≥ 0.30?} + O -->|yes| P[Select via LOCAL FILE copy
+ write per-stream cache
toast: Matched · N%] + O -->|no| Q[Fallback ladder] + O2 -->|yes| P + O2 -->|no| Q + N --> Q + Q --> R{AI translating on screen?} + R -->|yes| S[Keep AI, re-resolve source
toast: keeping AI translation + best N%] + R -->|no| R2{AI feature available?
master on + key + source
NOT gated on auto-select} + R2 -->|yes| S2[Activate AI translation
toast: using AI translation] + R2 -->|no| T{Current selection already
in preferred language?} + T -->|yes| U[Keep it
toast: best N%] + T -->|no| V[Select top release-name candidate
toast: sync unverified] +``` + +Fallback rationale: AI timing comes from the built-in track (verified), so it beats any +unverified addon pick. Auto-select only governs *unprompted* activation at playback start; a +failed scan explicitly asked for the best available subtitle. Same ladder applies when zero +target-language candidates exist. + +### Timing scan (reference collection loop, 300ms ticks) + +- Merge **buffered upcoming cues** (reflection into the text renderer; typically only 2–12 + visible) with **realtime rendered-cue intervals** (`onPlayerCues`), deduped by overlap. + ⚠ media3 picks a cue resolver per the track's cue-replacement behavior. For REPLACE tracks + (`ReplacingCuesResolver`, e.g. MKV SubRip) the `CuesWithTiming.durationUs` is **C.TIME_UNSET** + (large negative) — each cue lasts until replaced. The interval converter must coerce + non-positive durations to a nominal ~2s, or every item is rejected (`end < start`) and the + buffered path silently returns 0 for that whole content category: scans crawl at realtime pace, + and realtime windows carry a systematic ~600ms render-lag skew that suppresses well-synced subs + to ≈ the 0.70 threshold. Realtime reference intervals ≠ authored cue times — buffered is the + trustworthy source. (When debugging extraction, temporarily log resolver class/field/item + counts inside `extractBufferedIntervals` — that's how both resolver bugs were found.) +- Guards on the buffered read: candidate-time-range sanity filter; **self-match guard by TEXT** — + sample up to 8 buffered cue texts and drop the read if ≥half match the previously displayed + candidate's own (normalized) lines. Guard history, do not regress: + 1. exact-coincidence timing (±60ms both edges) — defeated by the reflection's uniform + stream-offset shift; a candidate scored 0.86 against itself; + 2. constant-delta timing (median ±80ms) — false-positived on subs cut from the same master + (Hebrew vs Czech share cue timings), discarding legit references and freezing scans at + realtime pace; + 3. **text comparison (current)** — language-discriminating, immune to both failure modes. + The track switch to the reference also propagates asynchronously: ~1.5s settle delay, target + indices re-resolved from current state, and the embedded override retries up to 10s with fresh + indices (one-shot appliers are fragile after MediaItem rebuilds), disabling text while invalid + instead of falling back to preferred-language (which silently kept the displayed sub rendering). +- Realtime intervals: capped at 20s each (longer = seek artifact), reset on backward seek. + +**Exit conditions** (first hit wins): + +| Condition | Values | +|---|---| +| Target reached | ≥ 8 intervals AND span ≥ 30s | +| Early accept | ≥ 4 intervals, span ≥ 30s, some candidate ≥ 0.8 | +| Fast accept | ≥ 5 intervals, span ≥ 15s, some candidate ≥ 0.85 | +| Give-up (no speech yet) | 4 min | +| Give-up (stagnation) | 90s since last new interval (clock **frozen while paused**) | +| Absolute ceiling | 10 min | + +Post-loop: `< 2` intervals or span `< 15s` ⇒ inconclusive (fall to ladder). + +### Scoring (`SubtitleSyncMatcher.scoreByTiming`) — HYBRID, calibrated + +Per reference window, averaged: +- Window ≤ 7s (a real cue): **best single-cue overlap** — this discriminates offsets; union + coverage here scored *everything* 0.9+ in dense dialogue (offset subs have cue chains touching + any short window). +- Window > 7s (realtime merges back-to-back cues): **union-of-cues coverage** — a synced sub + needs several cues to span it; single-cue unfairly caps at ~0.3. + +Calibration (user-verified, July 2026): synced subs score 0.71–0.98; a confirmed-offset sub +scored 0.65. Accept threshold **0.70** (timing) / **0.30** (hearing). The good/bad boundary is +narrow — do not nudge thresholds or the metric without fresh A/B evidence. + +--- + +## 3. Selection & the local-file trick + +Selecting an **external** subtitle rebuilds the MediaItem (re-prepare). Economics: +- Video re-open: debrid streams are deliberately **not disk-cached** (I/O bottleneck) — usually + fine, occasionally slow/stale. +- Subtitle download during rebuild was the common stall (slow addon proxies) ⇒ matched subs are + served from a **local cache file** (`cacheDir/matched_subs/`, ≤40 files): + the scan already downloaded the text; `localizeSubtitle()` writes UTF-8 (gunzipped) and selects + a `file://` copy (same id). The data source chain wraps OkHttp in `DefaultDataSource` for + file support. The tracks-changed remap keeps `file:` selections as-is (would otherwise swap + back to the remote URL and re-trigger a rebuild). +- **Do NOT preload all subs into the MediaItem** — tried historically and reverted: ExoPlayer + eagerly downloads every side-loaded config at prepare (30+ requests), plus encoding issues. + +Subtitle MIME sniffing strips trailing slashes (`…/sub.vtt/?q=` is VTT — AIOStreams shape); +default is SRT for extensionless URLs. + +## 4. Per-stream remembered cache + +- Key = stream identity, **not** title: `infoHash:fileIdx` → `videoHash` → `filename:size` → + URL hash. Same episode from a different source ⇒ different key ⇒ fresh scan (sync is a property + of the file). +- Value = candidate `provider|id` (not URL — addon URLs are ephemeral). Stored as JSON list in + `subtitle_match_cache_v1` (global DataStore), LRU 50. **Not cloud-synced** (deliberate). +- Written on verified match only (never for unverified fallback picks). +- Manual track pick in the target language **overwrites** the entry; picking anything else + clears it. Manual "Find Best Match" click clears it and scans fresh (`useCache=false`). + +## 5. State resets (bugs lived here) + +- `loadMedia`: resets all selection flags, `autoMatchAttempted`, AI source, **and the subtitles + list** — externals accumulate additively during one video; without clearing, a new video scans + the previous title's candidates (references vs wrong-show subs ⇒ uniform ~0.1 scores while the + correct fresh sub isn't scanned at all). +- Provided-URL loads with **no resolvable IMDb id** must still run `scheduleSubtitleSelection` + (embedded tracks + AI exist without addon subs). The silent else-branch here was one cause of + the "plays with subtitles Off, no scan, nothing" regression. +- The deeper cause: the subtitle fetch/selection block was the **last child coroutine of the + load job** — an uncaught exception in ANY sibling (VOD/home-server/stream appenders) cancels + all siblings, killing the subtitle flow before its first line. It now runs on `viewModelScope` + directly (as `subtitleRefreshJob`, cancelled by `loadMedia`), and each sibling is + failure-isolated with a `load child '' failed` log. Rule: anything subtitle-critical + must not share a Job with flaky background appenders. +- `selectStream` (mid-session source switch — does NOT pass through `loadMedia`): must reset + selection flags, purge **embedded** subtitle entries (per-file!), clear selection with a nonce + bump, cancel a running scan. Missing pieces here caused: wrong-track selection (stale indices), + "selected but not rendering" (no nonce bump), auto-flow never re-running (stale + `hasManualSubtitleSelection`). +- During the scan, subtitles are hidden (`subtitleView.visibility`) when + `isFindingBestMatch && !isAiTranslating` — display-only; cue collection listens on the player. +- Startup watchdog failover must reset `streamSelectedTime` optimistically before `continue`, + or it burns the whole source list in milliseconds (stale-clock burst). + +## 6. AI specifics + +- Batch translation pre-fetch (`triggerPreTranslation`/`preTranslateWindow`) is gated on + `translationManager.isEnabled` — without this it spends API requests whenever any track renders + (the 401-toast-with-AI-off bug). +- AI interim during scans: only when `aiSubtitleEnabled && aiSubtitleAutoSelect`. Source is + re-resolved on every activation (embedded preferred; external only if English) — a stale + external source caused mistimed translations. +- Hearing fallback: Gemini model + key + AI on; aborts on WS ERROR; same progress-anchored + timers as the timing scan. +- Gemini Live target language comes from the preference (`targetLanguageCode`), default `he`. + +## 7. Provider quirks (see also memory: project_aiostreams) + +- **AIOStreams**: subtitles endpoint slow (8–12s) / 502s cold → subtitle fetch is parallel, + 30s timeout, one retry. Its streams are exempt from quality sorting (`keepsOwnStreamOrder`) — + user pre-sorts server-side. Sub URLs end `…/sub.vtt/`. +- **OpenSubtitles v3+ proxy** (`opensubtitles.stremio.homes`): VTT with double `WEBVTT` header, + a branding banner cue at 0:01–0:06, recap lines as NOTE comments. Scanner's parser is tolerant; + keep an eye on ExoPlayer's stricter parser if display anomalies appear. + +## 8. Known-good log workflow + +Debug via `adb logcat` tag `SubMatch`: +- `reference source=…` — scan entered scoring (after source-wait). +- `align src=buffer|realtime|buffer+realtime buffered=N total=M refs=[…] candNear=[…]` — + reference intervals vs the first candidate's cues nearest the scan window; `src=buffer` + + score **exactly 1.00** = self-match red flag; `buffered=0` = extraction dead (see §2 warning). +- `[builtin] candidate … score=X` / `[hearing] candidate …` — per-candidate verdicts. +- Rare warnings that flag real faults: `realtime self-cue interval dropped` (reference track + switch didn't land), `subtitle fetch skipped: no imdbId`, `load child '' failed`, + `hearing aborted`, `match cache write failed`. +Toast decoder: "Matched · N%" = verified; "(remembered)" = cache hit; "(sync unverified)" = +fallback pick; "No well-synced … (best N%)" = rejection with evidence.