Fix broken playback of video files containing both Dolby Vision and HDR10+

android/app/build.gradle

@@ -140,6 +140,8 @@ dependencies {
     implementation("org.jellyfin.media3:media3-ffmpeg-decoder:$media3_version")
     implementation("io.github.peerless2012:ass-media:0.3.0")
 
+    testImplementation("junit:junit:4.13.2")
+
     //UI
     implementation("io.github.rabehx:iconsax-compose:0.0.5")
     implementation("io.coil-kt.coil3:coil-compose:3.3.0")

android/app/src/main/kotlin/nl/jknaapen/fladder/player/DvBitstreamSanitizer.kt

@@ -0,0 +1,113 @@
+package nl.jknaapen.fladder.player
+
+import java.nio.ByteBuffer
+
+/**
+ * In-place sanitizer for HEVC Annex B buffers carrying both Dolby Vision and HDR10+
+ * dynamic metadata. Some Android TV chipsets fail when a native DV decoder also
+ * receives in-band HDR10+ SEI, so keep only the dynamic metadata for the selected
+ * decode path.
+ */
+object DvBitstreamSanitizer {
+    private const val NAL_TYPE_PREFIX_SEI = 39
+    private const val NAL_TYPE_SUFFIX_SEI = 40
+    private const val NAL_TYPE_UNSPEC62 = 62
+    private const val NAL_TYPE_UNSPEC63 = 63
+
+    private const val SEI_PAYLOAD_TYPE_ITU_T_T35 = 4
+
+    fun sanitize(data: ByteBuffer, stripHdr10PlusSei: Boolean, stripDvRpu: Boolean) {
+        val startPos = data.position()
+        val limit = data.limit()
+        var writePos = startPos
+        var nalStartIndex = -1
+        var startCodeLen = 0
+
+        var i = startPos
+        while (i <= limit) {
+            val atEnd = i == limit
+            var foundStartCode = false
+            var nextStartCodeLen = 0
+            if (!atEnd && i + 2 < limit && data.get(i).toInt() == 0 && data.get(i + 1).toInt() == 0) {
+                if (data.get(i + 2).toInt() == 1) {
+                    foundStartCode = true
+                    nextStartCodeLen = 3
+                } else if (data.get(i + 2).toInt() == 0 && i + 3 < limit && data.get(i + 3).toInt() == 1) {
+                    foundStartCode = true
+                    nextStartCodeLen = 4
+                }
+            }
+
+            if (foundStartCode || atEnd) {
+                if (nalStartIndex >= 0) {
+                    val nalDataStart = nalStartIndex + startCodeLen
+                    val nalEnd = i
+                    var strip = false
+
+                    if (nalEnd - nalDataStart >= 2) {
+                        val nalUnitType = (data.get(nalDataStart).toInt() and 0x7E) shr 1
+                        strip = when (nalUnitType) {
+                            NAL_TYPE_UNSPEC62, NAL_TYPE_UNSPEC63 -> stripDvRpu
+                            NAL_TYPE_PREFIX_SEI, NAL_TYPE_SUFFIX_SEI ->
+                                stripHdr10PlusSei && isHdr10PlusSeiNalUnit(data, nalDataStart + 2, nalEnd)
+                            else -> false
+                        }
+                    }
+
+                    if (!strip) {
+                        if (writePos != nalStartIndex) {
+                            for (j in nalStartIndex until nalEnd) {
+                                data.put(writePos++, data.get(j))
+                            }
+                        } else {
+                            writePos = nalEnd
+                        }
+                    }
+                }
+                nalStartIndex = i
+                startCodeLen = nextStartCodeLen
+                i += if (nextStartCodeLen > 0) nextStartCodeLen else 1
+            } else {
+                i++
+            }
+        }
+
+        data.limit(writePos)
+        data.position(startPos)
+    }
+
+    private fun isHdr10PlusSeiNalUnit(data: ByteBuffer, rbspStart: Int, nalEnd: Int): Boolean {
+        var pos = rbspStart
+        if (pos >= nalEnd) return false
+
+        var payloadType = 0
+        while (pos < nalEnd) {
+            val value = data.get(pos++).toInt() and 0xFF
+            payloadType += value
+            if (value != 0xFF) break
+        }
+
+        var payloadSize = 0
+        while (pos < nalEnd) {
+            val value = data.get(pos++).toInt() and 0xFF
+            payloadSize += value
+            if (value != 0xFF) break
+        }
+
+        if (payloadType != SEI_PAYLOAD_TYPE_ITU_T_T35 || payloadSize < 7 || pos + 7 > nalEnd) {
+            return false
+        }
+
+        val countryCode = data.get(pos).toInt() and 0xFF
+        val providerCode = ((data.get(pos + 1).toInt() and 0xFF) shl 8) or (data.get(pos + 2).toInt() and 0xFF)
+        val orientedCode = ((data.get(pos + 3).toInt() and 0xFF) shl 8) or (data.get(pos + 4).toInt() and 0xFF)
+        val appIdentifier = data.get(pos + 5).toInt() and 0xFF
+        val appVersion = data.get(pos + 6).toInt() and 0xFF
+
+        return countryCode == 0xB5 &&
+            providerCode == 0x003C &&
+            orientedCode == 0x0001 &&
+            appIdentifier == 4 &&
+            (appVersion == 0 || appVersion == 1)
+    }
+}

android/app/src/main/kotlin/nl/jknaapen/fladder/player/DvSanitizingRenderersFactory.kt

@@ -0,0 +1,107 @@
+package nl.jknaapen.fladder.player
+
+import android.content.Context
+import android.os.Handler
+import android.util.Log
+import androidx.annotation.OptIn
+import androidx.media3.common.MimeTypes
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.decoder.DecoderInputBuffer
+import androidx.media3.exoplayer.DefaultRenderersFactory
+import androidx.media3.exoplayer.Renderer
+import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter
+import androidx.media3.exoplayer.mediacodec.MediaCodecSelector
+import androidx.media3.exoplayer.video.MediaCodecVideoRenderer
+import androidx.media3.exoplayer.video.VideoRendererEventListener
+
+private const val TAG = "FladderPlayer"
+
+@OptIn(UnstableApi::class)
+class DvSanitizingRenderersFactory(context: Context) : DefaultRenderersFactory(context) {
+    override fun buildVideoRenderers(
+        context: Context,
+        extensionRendererMode: Int,
+        mediaCodecSelector: MediaCodecSelector,
+        enableDecoderFallback: Boolean,
+        eventHandler: Handler,
+        eventListener: VideoRendererEventListener,
+        allowedVideoJoiningTimeMs: Long,
+        out: ArrayList<Renderer>
+    ) {
+        super.buildVideoRenderers(
+            context,
+            extensionRendererMode,
+            mediaCodecSelector,
+            enableDecoderFallback,
+            eventHandler,
+            eventListener,
+            allowedVideoJoiningTimeMs,
+            out
+        )
+
+        val rendererIndex = out.indexOfFirst { it.javaClass == MediaCodecVideoRenderer::class.java }
+        if (rendererIndex < 0) return
+
+        out[rendererIndex] = DvSanitizingVideoRenderer(
+            MediaCodecVideoRenderer.Builder(context)
+                .setCodecAdapterFactory(codecAdapterFactory)
+                .setMediaCodecSelector(mediaCodecSelector)
+                .setAllowedJoiningTimeMs(allowedVideoJoiningTimeMs)
+                .setEnableDecoderFallback(enableDecoderFallback)
+                .setEventHandler(eventHandler)
+                .setEventListener(eventListener)
+                .setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
+        )
+    }
+}
+
+@OptIn(UnstableApi::class)
+private class DvSanitizingVideoRenderer(builder: MediaCodecVideoRenderer.Builder) : MediaCodecVideoRenderer(builder) {
+    private var stripHdr10PlusSei = false
+    private var stripDvRpu = false
+
+    override fun onCodecInitialized(
+        name: String,
+        configuration: MediaCodecAdapter.Configuration,
+        initializedTimestampMs: Long,
+        initializationDurationMs: Long
+    ) {
+        super.onCodecInitialized(name, configuration, initializedTimestampMs, initializationDurationMs)
+
+        val codecs = configuration.format.codecs?.lowercase() ?: ""
+        val dvHevcFormat = configuration.format.sampleMimeType == MimeTypes.VIDEO_DOLBY_VISION &&
+            (codecs.startsWith("dvhe.") || codecs.startsWith("dvh1."))
+        val codecMimeType = configuration.codecInfo.codecMimeType
+        val newStripHdr10PlusSei = dvHevcFormat && codecMimeType == MimeTypes.VIDEO_DOLBY_VISION
+        val newStripDvRpu = dvHevcFormat &&
+            codecMimeType == MimeTypes.VIDEO_H265 &&
+            isBlCompatibleDvProfile(codecs)
+
+        if (newStripHdr10PlusSei != stripHdr10PlusSei || newStripDvRpu != stripDvRpu) {
+            Log.i(
+                TAG,
+                "DV bitstream sanitizing: stripHdr10PlusSei=$newStripHdr10PlusSei, " +
+                    "stripDvRpu=$newStripDvRpu (codec=$name, codecs=${configuration.format.codecs})"
+            )
+        }
+
+        stripHdr10PlusSei = newStripHdr10PlusSei
+        stripDvRpu = newStripDvRpu
+    }
+
+    override fun onQueueInputBuffer(buffer: DecoderInputBuffer) {
+        if (stripHdr10PlusSei || stripDvRpu) {
+            val data = buffer.data
+            if (data != null && data.hasRemaining() && !buffer.isEncrypted) {
+                DvBitstreamSanitizer.sanitize(data, stripHdr10PlusSei, stripDvRpu)
+            }
+        }
+        super.onQueueInputBuffer(buffer)
+    }
+
+    private fun isBlCompatibleDvProfile(codecs: String): Boolean =
+        codecs.startsWith("dvhe.07") ||
+            codecs.startsWith("dvh1.07") ||
+            codecs.startsWith("dvhe.08") ||
+            codecs.startsWith("dvh1.08")
+}

android/app/src/main/kotlin/nl/jknaapen/fladder/player/ExoPlayer.kt

@@ -90,7 +90,7 @@ internal fun ExoPlayer(
         .setContentType(C.AUDIO_CONTENT_TYPE_MOVIE)
         .build()
 
-    val renderersFactory = DefaultRenderersFactory(context)
+    val renderersFactory = DvSanitizingRenderersFactory(context)
         .setExtensionRendererMode(DefaultRenderersFactory.EXTENSION_RENDERER_MODE_ON)
         .setEnableDecoderFallback(true)
 

android/app/src/test/kotlin/nl/jknaapen/fladder/player/DvBitstreamSanitizerTest.kt

@@ -0,0 +1,131 @@
+package nl.jknaapen.fladder.player
+
+import java.nio.ByteBuffer
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class DvBitstreamSanitizerTest {
+    @Test
+    fun stripsHdr10PlusPrefixSeiBetweenVclNals() {
+        val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02))
+        val vcl2 = annexBNal(1, byteArrayOf(0x03, 0x04))
+        val buffer = bufferOf(vcl1, hdr10PlusSei(), vcl2)
+        val originalLimit = buffer.limit()
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
+        assertTrue(buffer.limit() < originalLimit)
+        assertEquals(0, buffer.position())
+    }
+
+    @Test
+    fun stripsSuffixSei() {
+        val vcl = annexBNal(1, byteArrayOf(0x01))
+        val suffixSei = annexBNal(40, hdr10PlusSeiPayload())
+        val buffer = bufferOf(vcl, suffixSei)
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertArrayEquals(vcl, remainingBytes(buffer))
+    }
+
+    @Test
+    fun handles3ByteStartCodes() {
+        val vcl1 = annexBNal(1, byteArrayOf(0x01, 0x02), startCodeLen = 3)
+        val sei = annexBNal(39, hdr10PlusSeiPayload(), startCodeLen = 3)
+        val vcl2 = annexBNal(1, byteArrayOf(0x03), startCodeLen = 3)
+        val buffer = bufferOf(vcl1, sei, vcl2)
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertArrayEquals(concat(vcl1, vcl2), remainingBytes(buffer))
+    }
+
+    @Test
+    fun preservesNonHdr10PlusT35Sei() {
+        val sei = annexBNal(
+            39,
+            byteArrayOf(0x04, 0x07, 0x00, 0x00, 0x3C, 0x00, 0x01, 0x04, 0x00)
+        )
+        val buffer = bufferOf(annexBNal(1, byteArrayOf(0x01)), sei, annexBNal(1, byteArrayOf(0x02)))
+        val original = remainingBytes(buffer)
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertArrayEquals(original, remainingBytes(buffer))
+    }
+
+    @Test
+    fun rpuModeStripsRpuAndElButKeepsHdr10PlusSei() {
+        val vcl = annexBNal(1, byteArrayOf(0x01))
+        val rpu = annexBNal(62, byteArrayOf(0x19, 0x08))
+        val el = annexBNal(63, byteArrayOf(0x42))
+        val sei = hdr10PlusSei()
+        val buffer = bufferOf(vcl, rpu, sei, el)
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = false, stripDvRpu = true)
+
+        assertArrayEquals(concat(vcl, sei), remainingBytes(buffer))
+    }
+
+    @Test
+    fun respectsPositionAndRestoresIt() {
+        val prefix = byteArrayOf(0xAA.toByte(), 0xBB.toByte())
+        val vcl = annexBNal(1, byteArrayOf(0x01))
+        val content = concat(prefix, vcl, hdr10PlusSei())
+        val buffer = ByteBuffer.wrap(content.copyOf())
+        buffer.position(prefix.size)
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertEquals(prefix.size, buffer.position())
+        assertArrayEquals(vcl, remainingBytes(buffer))
+        assertEquals(0xAA.toByte(), buffer.get(0))
+        assertEquals(0xBB.toByte(), buffer.get(1))
+    }
+
+    @Test
+    fun worksOnDirectBuffers() {
+        val vcl = annexBNal(1, byteArrayOf(0x01, 0x02))
+        val content = concat(vcl, hdr10PlusSei())
+        val buffer = ByteBuffer.allocateDirect(content.size)
+        buffer.put(content)
+        buffer.flip()
+
+        DvBitstreamSanitizer.sanitize(buffer, stripHdr10PlusSei = true, stripDvRpu = false)
+
+        assertArrayEquals(vcl, remainingBytes(buffer))
+    }
+
+    private fun annexBNal(nalUnitType: Int, payload: ByteArray, startCodeLen: Int = 4): ByteArray {
+        val startCode = if (startCodeLen == 3) byteArrayOf(0, 0, 1) else byteArrayOf(0, 0, 0, 1)
+        val header = byteArrayOf(((nalUnitType shl 1) and 0x7E).toByte(), 0x01)
+        return concat(startCode, header, payload)
+    }
+
+    private fun hdr10PlusSeiPayload(): ByteArray =
+        byteArrayOf(0x04, 0x07, 0xB5.toByte(), 0x00, 0x3C, 0x00, 0x01, 0x04, 0x00, 0x80.toByte())
+
+    private fun hdr10PlusSei(): ByteArray = annexBNal(39, hdr10PlusSeiPayload())
+
+    private fun concat(vararg parts: ByteArray): ByteArray {
+        val result = ByteArray(parts.sumOf { it.size })
+        var offset = 0
+        for (part in parts) {
+            part.copyInto(result, offset)
+            offset += part.size
+        }
+        return result
+    }
+
+    private fun bufferOf(vararg parts: ByteArray): ByteBuffer = ByteBuffer.wrap(concat(*parts))
+
+    private fun remainingBytes(buffer: ByteBuffer): ByteArray {
+        val copy = ByteArray(buffer.remaining())
+        buffer.duplicate().get(copy)
+        return copy
+    }
+}