From 8b209d1e82a1e367a72db1d2e8f2d590109aa028 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 13 Sep 2026 17:50:00 +0200 Subject: [PATCH 001/137] feat: forward remote login diagnostics to the client --- .../services/YoutubeRemoteBrowserMessageGuard.kt | 3 ++- .../server/YoutubeRemoteBrowserMessageGuardTest.kt | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt index 3c9e475c..31ca2538 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt @@ -12,7 +12,7 @@ import kotlinx.serialization.json.jsonPrimitive object YoutubeRemoteBrowserMessageGuard { private val json = Json { ignoreUnknownKeys = true } private val clientTypes = setOf("resize", "pointer", "wheel", "key", "text", "cancel") - private val tokenTypes = setOf("status", "error") + private val tokenTypes = setOf("status", "error", "log") private val phases = setOf("opening", "awaiting_login", "capturing_session", "connected") private val pointerEvents = setOf("down", "up", "move") private val keyEvents = setOf("down", "up") @@ -45,6 +45,7 @@ object YoutubeRemoteBrowserMessageGuard { when (obj.string("type")?.takeIf { it in tokenTypes }) { "status" -> obj.string("phase")?.let { it in phases } == true "error" -> (obj.string("message")?.length ?: Int.MAX_VALUE) <= 200 + "log" -> obj.int("at").inRange(0..Int.MAX_VALUE) && (obj.string("message")?.length ?: Int.MAX_VALUE) <= 1000 else -> false } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt index 2469785f..ad58db12 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt @@ -30,4 +30,14 @@ class YoutubeRemoteBrowserMessageGuardTest { assertNull(YoutubeRemoteBrowserMessageGuard.tokenText(complete)) assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"status","phase":"unknown"}""")) } + + @Test + fun `token diagnostics logs are forwarded when bounded`() { + val log = """{"type":"log","at":1200,"message":"login check url=https://accounts.google.com/signin cookies=google.com=[SID]"}""" + + assertEquals(log, YoutubeRemoteBrowserMessageGuard.tokenText(log)) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","message":"missing at"}""")) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","at":-1,"message":"negative"}""")) + assertNull(YoutubeRemoteBrowserMessageGuard.tokenText("""{"type":"log","at":5,"message":"${"x".repeat(1001)}"}""")) + } } From 82d659d8886e7389dd8c758abb3251af1c2cfe67 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 13 Sep 2026 17:50:00 +0200 Subject: [PATCH 002/137] feat: log remote login bridge and completion outcomes --- .../OkHttpYoutubeRemoteBrowserClient.kt | 34 +++---------- .../services/YoutubeRemoteBrowserAudit.kt | 46 +++++++++++++++++ .../YoutubeRemoteBrowserBridgeListener.kt | 50 +++++++++++++++++++ .../YoutubeRemoteBrowserCompleteResult.kt | 8 +++ .../services/YoutubeRemoteBrowserService.kt | 25 ++++------ 5 files changed, 121 insertions(+), 42 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt index 4a321af8..0c485ee2 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt @@ -15,9 +15,6 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response -import okhttp3.WebSocket -import okhttp3.WebSocketListener -import okio.ByteString class OkHttpYoutubeRemoteBrowserClient( private val serviceUrl: String, @@ -55,12 +52,17 @@ class OkHttpYoutubeRemoteBrowserClient( ): Unit = coroutineScope { val done = CompletableDeferred() val outbound = Channel(config.outboundQueueSize, BufferOverflow.DROP_OLDEST) - val socket = client.newWebSocket(tokenWebSocketRequest(tokenSessionId, internalToken), listener(outbound, done, config)) + val listener = YoutubeRemoteBrowserBridgeListener(tokenSessionId, outbound, done, config) + val socket = client.newWebSocket(tokenWebSocketRequest(tokenSessionId, internalToken), listener) + var inputs = 0 val outboundJob = launch { for (frame in outbound) serverSession.send(frame) } val inboundJob = launch { for (frame in serverSession.incoming) { if (frame is Frame.Text) { - YoutubeRemoteBrowserMessageGuard.frontendText(frame.readText(), config.maxInputBytes)?.let(socket::send) + YoutubeRemoteBrowserMessageGuard.frontendText(frame.readText(), config.maxInputBytes)?.let { + inputs += 1 + socket.send(it) + } } if (frame is Frame.Close) done.complete(Unit) } @@ -72,6 +74,7 @@ class OkHttpYoutubeRemoteBrowserClient( outbound.close() outboundJob.cancel() inboundJob.cancel() + YoutubeRemoteBrowserAudit.bridgeEnded(tokenSessionId, listener.texts.get(), listener.frames.get(), inputs) } private fun decodeStartResponse(response: Response): YoutubeRemoteBrowserTokenStartResponse? { @@ -85,27 +88,6 @@ class OkHttpYoutubeRemoteBrowserClient( .header(INTERNAL_HEADER, internalToken) .build() - private fun listener( - outbound: Channel, - done: CompletableDeferred, - config: YoutubeRemoteBrowserConfig, - ): WebSocketListener = object : WebSocketListener() { - override fun onMessage(webSocket: WebSocket, text: String) { - YoutubeRemoteBrowserMessageGuard.tokenText(text)?.let { outbound.trySend(Frame.Text(it)) } - } - - override fun onMessage(webSocket: WebSocket, bytes: ByteString) { - if (bytes.size <= config.maxFrameBytes) outbound.trySend(Frame.Binary(true, bytes.toByteArray())) - } - - override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - done.complete(Unit) - } - - override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - done.complete(Unit) - } - } private fun webSocketBaseUrl(): String = serviceUrl.trimEnd('/').replaceFirst("https://", "wss://").replaceFirst("http://", "ws://") diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt new file mode 100644 index 00000000..5ec135fc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.YoutubeRemoteBrowserCompleteRequest +import org.slf4j.LoggerFactory + +object YoutubeRemoteBrowserAudit { + private val log = LoggerFactory.getLogger("YoutubeRemoteLogin") + private val typePattern = Regex("\"type\"\\s*:\\s*\"([a-z_]{1,32})\"") + + fun startResult(userId: String, result: String) { + log.info("remote-login start user={} result={}", short(userId), result) + } + + fun bridgeOpened(tokenSessionId: String) { + log.info("remote-login bridge session={} token socket open", short(tokenSessionId)) + } + + fun bridgeClosed(tokenSessionId: String, code: Int, reason: String) { + log.info("remote-login bridge session={} token socket closed code={} reason={}", short(tokenSessionId), code, reason) + } + + fun bridgeFailed(tokenSessionId: String, error: Throwable) { + log.warn("remote-login bridge session={} token socket failed: {}", short(tokenSessionId), error.message ?: error.javaClass.simpleName) + } + + fun bridgeEnded(tokenSessionId: String, texts: Int, frames: Int, inputs: Int) { + log.info("remote-login bridge session={} ended texts={} frames={} inputs={}", short(tokenSessionId), texts, frames, inputs) + } + + fun droppedTokenMessage(tokenSessionId: String, text: String) { + val type = typePattern.find(text)?.groupValues?.get(1) ?: "unknown" + log.warn("remote-login bridge session={} dropped token message type={} bytes={}", short(tokenSessionId), type, text.length) + } + + fun completion(request: YoutubeRemoteBrowserCompleteRequest, result: YoutubeRemoteBrowserCompleteResult) { + val lines = request.cookies.lineSequence().count { it.isNotBlank() && !it.startsWith("#") } + val sapisid = request.cookies.contains("SAPISID") + val loginInfo = request.cookies.contains("LOGIN_INFO") + log.info( + "remote-login completion session={} result={} cookies={} sapisid={} loginInfo={} poToken={} authUser={}", + short(request.sessionId), result::class.simpleName, lines, sapisid, loginInfo, request.poToken.length, request.authUser, + ) + } + + private fun short(value: String): String = value.take(8) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt new file mode 100644 index 00000000..8580388f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt @@ -0,0 +1,50 @@ +package dev.typetype.server.services + +import io.ktor.websocket.Frame +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.channels.Channel +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import java.util.concurrent.atomic.AtomicInteger + +class YoutubeRemoteBrowserBridgeListener( + private val tokenSessionId: String, + private val outbound: Channel, + private val done: CompletableDeferred, + private val config: YoutubeRemoteBrowserConfig, +) : WebSocketListener() { + val texts = AtomicInteger() + val frames = AtomicInteger() + + override fun onOpen(webSocket: WebSocket, response: Response) { + YoutubeRemoteBrowserAudit.bridgeOpened(tokenSessionId) + } + + override fun onMessage(webSocket: WebSocket, text: String) { + val accepted = YoutubeRemoteBrowserMessageGuard.tokenText(text) + if (accepted == null) { + YoutubeRemoteBrowserAudit.droppedTokenMessage(tokenSessionId, text) + return + } + texts.incrementAndGet() + outbound.trySend(Frame.Text(accepted)) + } + + override fun onMessage(webSocket: WebSocket, bytes: ByteString) { + if (bytes.size > config.maxFrameBytes) return + frames.incrementAndGet() + outbound.trySend(Frame.Binary(true, bytes.toByteArray())) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + YoutubeRemoteBrowserAudit.bridgeClosed(tokenSessionId, code, reason) + done.complete(Unit) + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + YoutubeRemoteBrowserAudit.bridgeFailed(tokenSessionId, t) + done.complete(Unit) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt index 06325041..eba375c5 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt @@ -8,3 +8,11 @@ sealed interface YoutubeRemoteBrowserCompleteResult { data object InvalidCredentials : YoutubeRemoteBrowserCompleteResult data object Unavailable : YoutubeRemoteBrowserCompleteResult } + +fun YoutubeSessionCompleteResult.toRemoteBrowserResult(): YoutubeRemoteBrowserCompleteResult = when (this) { + YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed + YoutubeSessionCompleteResult.InvalidCode, + YoutubeSessionCompleteResult.ExpiredCode, + YoutubeSessionCompleteResult.InvalidCredentials -> YoutubeRemoteBrowserCompleteResult.InvalidCredentials + YoutubeSessionCompleteResult.Unavailable -> YoutubeRemoteBrowserCompleteResult.Unavailable +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt index a5d6d429..f65c6d1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt @@ -22,11 +22,13 @@ class YoutubeRemoteBrowserService( return YoutubeRemoteBrowserStartResult.Misconfigured } val internalToken = config.internalToken ?: return YoutubeRemoteBrowserStartResult.Misconfigured - return when (val reserved = sessions.reserve(userId, config)) { + val result = when (val reserved = sessions.reserve(userId, config)) { YoutubeRemoteBrowserReserveResult.AlreadyActive -> YoutubeRemoteBrowserStartResult.AlreadyActive YoutubeRemoteBrowserReserveResult.CapacityReached -> YoutubeRemoteBrowserStartResult.CapacityReached is YoutubeRemoteBrowserReserveResult.Reserved -> startTokenSession(reserved, userId, internalToken, returnTo) } + YoutubeRemoteBrowserAudit.startResult(userId, result::class.simpleName ?: "unknown") + return result } suspend fun cancel(userId: String, sessionId: String): Boolean { @@ -44,21 +46,12 @@ class YoutubeRemoteBrowserService( if (request.status != "completed") return YoutubeRemoteBrowserCompleteResult.InvalidPayload if (!youtubeSessionService.isConfigured) return YoutubeRemoteBrowserCompleteResult.Unavailable val session = sessions.complete(request.sessionId, request.tokenSessionId) - ?: return YoutubeRemoteBrowserCompleteResult.NotFound - return when ( - youtubeSessionService.completeRemote( - session.userId, - request.cookies, - request.poToken, - request.authUser, - ) - ) { - YoutubeSessionCompleteResult.Completed -> YoutubeRemoteBrowserCompleteResult.Completed - YoutubeSessionCompleteResult.InvalidCode, - YoutubeSessionCompleteResult.ExpiredCode, - YoutubeSessionCompleteResult.InvalidCredentials -> YoutubeRemoteBrowserCompleteResult.InvalidCredentials - YoutubeSessionCompleteResult.Unavailable -> YoutubeRemoteBrowserCompleteResult.Unavailable - } + ?: return YoutubeRemoteBrowserCompleteResult.NotFound.also { YoutubeRemoteBrowserAudit.completion(request, it) } + val result = youtubeSessionService + .completeRemote(session.userId, request.cookies, request.poToken, request.authUser) + .toRemoteBrowserResult() + YoutubeRemoteBrowserAudit.completion(request, result) + return result } suspend fun bridge(sessionId: String, wsToken: String?, serverSession: DefaultWebSocketServerSession): Unit { From 076634f78eeb041e614617fadef13334a47f19f3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 12:23:31 +0200 Subject: [PATCH 003/137] fix: recover cold SABR seeks --- .../dev/typetype/server/sabr/SabrSession.kt | 1 + .../services/SabrDemandAttemptFinisher.kt | 2 +- .../services/SabrPlaybackSegmentSelection.kt | 5 +- .../server/services/SabrPumpRuntime.kt | 13 ++-- .../server/services/SabrSessionHolder.kt | 1 + .../services/SabrDemandFailurePumpTest.kt | 2 + .../services/SabrMissingDemandRecoveryTest.kt | 33 ++++++++++ .../SabrPlaybackSessionServiceTest.kt | 5 ++ .../server/services/SabrPumpRuntimeTest.kt | 19 ++++-- .../services/SabrSeekDiagnosticsTest.kt | 62 +++++++++++++++++++ .../services/SabrSessionPumpLoopTest.kt | 1 + .../server/services/SabrSessionPumpTest.kt | 3 + .../services/SabrSessionTimeRequestsTest.kt | 36 ++++++++++- .../SabrTransientDemandFailureTest.kt | 2 + 14 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt index 314820ae..587054c4 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt +++ b/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt @@ -85,5 +85,6 @@ internal class YoutubeSabrSession( ) { val segmentCount: Int get() = delegate.segmentCount val targetTrackSegmentCount: Int get() = delegate.targetTrackSegmentCount + val requestPerformed: Boolean get() = delegate.wasRequestPerformed() } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index e4eccca5..2727597c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -68,7 +68,7 @@ internal object SabrDemandAttemptFinisher { } val action = runtime.demandRecoveryAction( requestKey = identity, - targetTrackSegmentCount = result.targetTrackSegmentCount, + requestPerformed = result.requestPerformed, resolved = resolved, ) val recovering = recover(holder, request, identity, action, runtime) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt index 5fa0feb9..93bc0fd8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt @@ -4,8 +4,11 @@ import dev.typetype.server.sabr.YoutubeSabrFormat internal fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { liveSequenceAt(format, playerTimeMs)?.let { return it } - return session.streamState.getSegmentNumberAtOrAfterTimeMs(format, playerTimeMs.coerceAtLeast(0L)) + val mappedSequence = session.streamState + .getSegmentNumberAtOrAfterTimeMs(format, playerTimeMs.coerceAtLeast(0L)) .coerceAtLeast(1) + val endSequence = session.streamState.getEndSegment(format).toInt() + return mappedSequence.coerceAtMost(endSequence.takeIf { it > 0 } ?: mappedSequence) } internal fun SabrSessionHolder.playbackContinuationSequence( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt b/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt index da984e00..fefd9b29 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt @@ -39,20 +39,17 @@ internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTi fun demandRecoveryAction( requestKey: String, - targetTrackSegmentCount: Int, + requestPerformed: Boolean, resolved: Boolean, ): SabrDemandRecoveryAction { - if (resolved) { + if (resolved || !requestPerformed) { resetDemandRecovery() return SabrDemandRecoveryAction.WAIT } ensureDemand(requestKey) - if (targetTrackSegmentCount > 0) { - if (!demandTrackReadvertised) { - demandTrackReadvertised = true - return SabrDemandRecoveryAction.READVERTISE_TRACK - } - return SabrDemandRecoveryAction.WAIT + if (!demandTrackReadvertised) { + demandTrackReadvertised = true + return SabrDemandRecoveryAction.READVERTISE_TRACK } return SabrDemandRecoveryAction.WAIT } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index 8bb0eb72..fb41f20c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -49,6 +49,7 @@ internal class SabrSessionHolder( setPlayerTimeMs(ms) val generation = activeGeneration.incrementAndGet() clearReaderStateBefore(generation) + SabrPlaybackDiagnostics.clear(this) pendingRefetch.set(null) pendingForwardSeek.set(null) for (itag in activeItags) { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt index 880dcb81..f131a4fb 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt @@ -138,6 +138,7 @@ class SabrDemandFailurePumpTest { val result = mockk() every { result.segmentCount } returns 7 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true return result } @@ -145,6 +146,7 @@ class SabrDemandFailurePumpTest { val result = mockk() every { result.segmentCount } returns 0 every { result.targetTrackSegmentCount } returns 0 + every { result.requestPerformed } returns true return result } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt index 84a84099..0eab1b8e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt @@ -33,6 +33,7 @@ class SabrMissingDemandRecoveryTest { every { streamState.getSegmentStartMs(audio, 44) } returns 429_337L every { result.segmentCount } returns 2 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true every { session.pumpOnceStreamingForDemand(any(), request) } returns result val holder = holder(session, audio, video) holder.requestSegmentDemand(request) @@ -48,6 +49,38 @@ class SabrMissingDemandRecoveryTest { } } + @Test + fun `companion-only response readvertises the missing target track`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val request = SabrSegmentRequest.media(audio, 44) + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + val result = mockk() + every { session.streamState } returns streamState + every { session.getCachedSegment(any()) } returns null + every { session.requestNumber } returns 25 + every { streamState.getMinBufferedEndMs() } returns 416_100L + every { streamState.getSegmentStartMs(audio, 44) } returns 429_337L + every { result.segmentCount } returns 1 + every { result.targetTrackSegmentCount } returns 0 + every { result.requestPerformed } returns true + every { session.pumpOnceStreamingForDemand(any(), request) } returns result + val holder = holder(session, audio, video) + holder.requestSegmentDemand(request) + var rounds = 0 + + SabrSessionPump().pumpLoop({ rounds++ == 0 }, holder, intervalMs = 100L) + + verify(exactly = 1) { session.prepareForMissingSegment(request) } + assertEquals("140:44", holder.pendingSegmentDemandSummary()) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + private fun holder( session: YoutubeSabrSession, audio: YoutubeSabrFormat, diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt index 83e37887..0987a39a 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt @@ -33,6 +33,7 @@ class SabrPlaybackSessionServiceTest { val prepared = SabrPreparedInfo(info, token()) val holder = holder(audio, video) val initializationFetches = AtomicInteger() + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(video, 88_168L) } answers { @@ -169,6 +170,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(any(), 90_000L) } returns 9 @@ -196,6 +198,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setSelectVideoFormatBeforeAudio(true) } returns Unit every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(video, 120_000L) } returns 24 @@ -220,6 +223,7 @@ class SabrPlaybackSessionServiceTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) + every { holder.session.streamState.getEndSegment(any()) } returns 0L every { holder.session.streamState.setActiveTrackTypes(any(), any()) } returns Unit every { holder.session.streamState.setSelectVideoFormatBeforeAudio(false) } returns Unit every { holder.session.streamState.getSegmentNumberAtOrAfterTimeMs(audio, 299L) } returns 1 @@ -248,6 +252,7 @@ class SabrPlaybackSessionServiceTest { every { session.awaitReadableSegment(any(), any()) } returns null every { session.isBeyondEnd(any()) } returns false every { session.prepareForInitialization(any()) } returns Unit + every { state.getEndSegment(any()) } returns 0L every { state.setActiveTrackTypes(any(), any()) } returns Unit every { state.setSelectVideoFormatBeforeAudio(any()) } returns Unit return SabrSessionHolder( diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt index 162d2c52..b0440c56 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt @@ -76,23 +76,32 @@ class SabrPumpRuntimeTest { } @Test - fun `slow response without media stays recoverable for watchdog`() { + fun `deferred request stays recoverable for watchdog`() { var now = 1_000L val runtime = SabrPumpRuntime { now } runtime.beginDemand("140:44") now += SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS - assertEquals(SabrDemandRecoveryAction.WAIT, runtime.demandRecoveryAction("140:44", 0, false)) + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = false, resolved = false), + ) } @Test - fun `repeated responses without demanded segment stay retryable`() { + fun `response without demanded segment is readvertised once`() { val runtime = SabrPumpRuntime { 1_000L } runtime.beginDemand("140:44") - assertEquals(SabrDemandRecoveryAction.READVERTISE_TRACK, runtime.demandRecoveryAction("140:44", 1, false)) + assertEquals( + SabrDemandRecoveryAction.READVERTISE_TRACK, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) repeat(5) { - assertEquals(SabrDemandRecoveryAction.WAIT, runtime.demandRecoveryAction("140:44", 1, false)) + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt new file mode 100644 index 00000000..d4198746 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt @@ -0,0 +1,62 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrSeekDiagnosticsTest { + @Test + fun `seek generation clears diagnostics from the previous position`() { + SabrSegmentDemandTracker.clearAll() + try { + val audio = format(140, true) + val video = format(137, false) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { state.getSegmentNumberAtOrAfterTimeMs(any(), 120_000L) } returns 1 + every { state.getMinBufferedEndMs() } returns 0L + val holder = holder(session, audio, video) + SabrPlaybackDiagnostics.record(holder, SabrSegmentRequest.media(audio, 1), "status=2 policy=true") + + SabrPlaybackSessionService(mockk(relaxed = true)).seekExisting(holder, 120_000L) + + assertNull(SabrPlaybackDiagnostics.blocker(holder)) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.audioTrackId } returns null + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + every { format.lastModified } returns 0L + every { format.xtags } returns "" + return format + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt index d6e708ec..8394c93f 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt @@ -270,6 +270,7 @@ class SabrSessionPumpLoopTest { val result = mockk() every { result.segmentCount } returns segmentCount every { result.targetTrackSegmentCount } returns targetTrackSegmentCount + every { result.requestPerformed } returns true return result } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt index e1209e0f..503e7361 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt @@ -145,6 +145,7 @@ class SabrSessionPumpTest { every { streamState.setActiveTrackTypes(true, true) } returns Unit every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 every { streamState.getSegmentNumberAtOrAfterTimeMs(audio, 60_862L) } returns 7 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 50_876L, 9_985L, sequence = 7)) @@ -162,6 +163,7 @@ class SabrSessionPumpTest { val session = mockk() val streamState = mockk() every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 59_907L, 9_985L, sequence = 7)) @@ -180,6 +182,7 @@ class SabrSessionPumpTest { every { session.streamState } returns streamState every { streamState.setActiveTrackTypes(true, true) } returns Unit every { streamState.getSegmentNumberAtOrAfterTimeMs(video, 60_862L) } returns 12 + every { streamState.getEndSegment(any()) } returns 0L val holder = sabrHolder(audio, video, session, streamState) holder.markServed(mediaSegment(137, 55_789L, 5_072L, sequence = 12)) holder.markServed(mediaSegment(140, 59_907L, 9_985L, sequence = 7)) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt index 55509e38..a8b9a821 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt @@ -14,12 +14,42 @@ import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSessionTimeRequestsTest { + @Test + fun `playback start sequence is clamped to the known final segment`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 91 + every { state.getEndSegment(audio) } returns 90L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(90, holder.playbackStartSequence(audio, 88_168L)) + } + + @Test + fun `playback start sequence stays valid without an indexed end`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 33 + every { state.getEndSegment(audio) } returns 0L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(33, holder.playbackStartSequence(audio, 88_168L)) + } + @Test fun `mediaRequestsAt returns active audio and video requests for player time`() { val audio = sabrFormat(itag = 140, isAudio = true) val video = sabrFormat(itag = 137, isAudio = false) val session = mockk() - val state = mockk() + val state = mockk(relaxed = true) every { session.streamState } returns state every { state.setActiveTrackTypes(any(), any()) } returns Unit every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 @@ -37,7 +67,7 @@ class SabrSessionTimeRequestsTest { val audio = sabrFormat(itag = 140, isAudio = true) val video = sabrFormat(itag = 247, isAudio = false) val session = mockk() - val state = mockk() + val state = mockk(relaxed = true) every { session.streamState } returns state every { state.setActiveTrackTypes(any(), any()) } returns Unit every { state.getSegmentNumberAtOrAfterTimeMs(video, 340_000L) } returns 64 @@ -55,7 +85,7 @@ class SabrSessionTimeRequestsTest { val audio = sabrFormat(itag = 140, isAudio = true) val video = sabrFormat(itag = 137, isAudio = false) val session = mockk() - val state = mockk() + val state = mockk(relaxed = true) every { session.streamState } returns state every { state.setActiveTrackTypes(any(), any()) } returns Unit every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt index cce09dc0..b4e4a134 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt @@ -77,6 +77,7 @@ class SabrTransientDemandFailureTest { every { streamState.getMinBufferedEndMs() } returns 379_233L every { result.segmentCount } returns 1 every { result.targetTrackSegmentCount } returns 1 + every { result.requestPerformed } returns true every { session.pumpOnceStreamingForDemand(any(), request) } answers { attempts++ if (attempts == 1) throw IOException("timeout") @@ -236,6 +237,7 @@ class SabrTransientDemandFailureTest { val result = mockk() every { result.segmentCount } returns segmentCount every { result.targetTrackSegmentCount } returns targetTrackSegmentCount + every { result.requestPerformed } returns true return result } From d987fd2a3cf178fee070fdf653b1ef4fb9ef4fde Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 13:04:03 +0200 Subject: [PATCH 004/137] feat: add presence access keys --- .../dev/typetype/server/db/DatabaseFactory.kt | 2 + .../server/db/tables/PresenceKeysTable.kt | 19 +++ .../typetype/server/models/PresenceModels.kt | 37 ++++++ .../server/services/PresenceKeyService.kt | 124 ++++++++++++++++++ .../server/services/PresenceTokenCodec.kt | 25 ++++ .../dev/typetype/server/TestDatabase.kt | 2 + 6 files changed, 209 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/PresenceModels.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 1368eb0f..97e1c770 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -33,6 +33,7 @@ import dev.typetype.server.db.tables.PushNotificationSeenVideosTable import dev.typetype.server.db.tables.PushNotificationEventsTable import dev.typetype.server.db.tables.PushNotificationDeliveriesTable import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.PresenceKeysTable import dev.typetype.server.db.tables.RecommendationEventsTable import dev.typetype.server.db.tables.RecommendationFeedHistoryTable import dev.typetype.server.db.tables.RecommendationFeedbackTable @@ -73,6 +74,7 @@ object DatabaseFactory { SchemaUtils.create( UsersTable, ProfileAccountsTable, + PresenceKeysTable, UserAvatarsTable, SessionsTable, AdminSettingsTable, diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt new file mode 100644 index 00000000..a84c9509 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object PresenceKeysTable : Table("presence_keys") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val tokenPrefix = text("token_prefix") + val tokenHash = text("token_hash").uniqueIndex() + val createdAt = long("created_at") + val lastUsedAt = long("last_used_at").nullable() + + init { + index(false, userId) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt b/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt new file mode 100644 index 00000000..a4b8ea1b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt @@ -0,0 +1,37 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class PresenceKeyCreateRequest( + val name: String? = null, +) + +@Serializable +data class PresenceKeyItem( + val id: String, + val name: String, + val tokenPrefix: String, + val scope: String = PresenceScopes.READ, + val createdAt: Long, + val lastUsedAt: Long? = null, +) + +@Serializable +data class PresenceKeyCreatedResponse( + val key: PresenceKeyItem, + val token: String, +) + +@Serializable +data class PresenceResponse( + val active: Boolean, + val nowPlaying: ActiveSessionNowPlayingItem? = null, + val retryAfterMs: Long = 15_000L, + val serverTimeMs: Long, +) + +object PresenceScopes { + const val READ = "presence:read" + const val PRESENCE_RETRY_MS = 15_000L +} diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt b/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt new file mode 100644 index 00000000..8aba8d37 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt @@ -0,0 +1,124 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PresenceKeysTable +import dev.typetype.server.models.PresenceKeyCreatedResponse +import dev.typetype.server.models.PresenceKeyItem +import java.util.UUID +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +data class PresenceTokenPrincipal( + val keyId: String, + val userId: String, +) + +class PresenceKeyLimitException(message: String) : IllegalStateException(message) + +class PresenceKeyService( + private val nowProvider: () -> Long = System::currentTimeMillis, +) { + suspend fun create(userId: String, requestedName: String?): PresenceKeyCreatedResponse { + val name = requestedName?.trim().orEmpty() + require(name.isNotEmpty() && name.length <= MAX_NAME_LENGTH) { + "Presence key name must contain 1 to $MAX_NAME_LENGTH characters" + } + return DatabaseFactory.query { + val activeCount = PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .count() + if (activeCount >= MAX_KEYS_PER_USER) { + throw PresenceKeyLimitException("Presence key limit reached") + } + val now = nowProvider() + val id = UUID.randomUUID().toString() + val token = PresenceTokenCodec.issue() + val tokenPrefix = PresenceTokenCodec.prefix(token) + PresenceKeysTable.insert { + it[PresenceKeysTable.id] = id + it[PresenceKeysTable.userId] = userId + it[PresenceKeysTable.name] = name + it[PresenceKeysTable.tokenPrefix] = tokenPrefix + it[PresenceKeysTable.tokenHash] = PresenceTokenCodec.hash(token) + it[createdAt] = now + it[lastUsedAt] = null + } + PresenceKeyCreatedResponse( + key = item(id, name, tokenPrefix, now, null), + token = token, + ) + } + } + + suspend fun list(userId: String): List = DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .sortedBy { it[PresenceKeysTable.createdAt] } + .map { + item( + id = it[PresenceKeysTable.id], + name = it[PresenceKeysTable.name], + tokenPrefix = it[PresenceKeysTable.tokenPrefix], + createdAt = it[PresenceKeysTable.createdAt], + lastUsedAt = it[PresenceKeysTable.lastUsedAt], + ) + } + } + + suspend fun revoke(userId: String, keyId: String): Boolean = DatabaseFactory.query { + PresenceKeysTable.deleteWhere { + PresenceKeysTable.id eq keyId + PresenceKeysTable.userId eq userId + } > 0 + } + + suspend fun resolve(token: String): PresenceTokenPrincipal? { + if (!token.startsWith(PresenceTokenCodec.PREFIX)) return null + val hash = PresenceTokenCodec.hash(token) + val now = nowProvider() + return DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.tokenHash eq hash } + .singleOrNull() + ?.let { + PresenceKeysTable.update({ PresenceKeysTable.id eq it[PresenceKeysTable.id] }) { + it[lastUsedAt] = now + } + PresenceTokenPrincipal( + keyId = it[PresenceKeysTable.id], + userId = it[PresenceKeysTable.userId], + ) + } + } + } + + suspend fun hasActiveKey(userId: String): Boolean = DatabaseFactory.query { + PresenceKeysTable.selectAll() + .where { PresenceKeysTable.userId eq userId } + .empty() + .not() + } + + private fun item( + id: String, + name: String, + tokenPrefix: String, + createdAt: Long, + lastUsedAt: Long?, + ) = PresenceKeyItem( + id = id, + name = name, + tokenPrefix = tokenPrefix, + scope = dev.typetype.server.models.PresenceScopes.READ, + createdAt = createdAt, + lastUsedAt = lastUsedAt, + ) + + private companion object { + const val MAX_KEYS_PER_USER = 10L + const val MAX_NAME_LENGTH = 50 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt b/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt new file mode 100644 index 00000000..1ac9340b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 + +internal object PresenceTokenCodec { + internal const val PREFIX = "ttp1_" + private val random = SecureRandom() + + fun issue(): String { + val bytes = ByteArray(32) + random.nextBytes(bytes) + return PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) + } + + fun prefix(token: String): String = token.take(PREFIX.length + 6) + + fun hash(token: String): String = Base64.getUrlEncoder().withoutPadding().encodeToString( + MessageDigest.getInstance("SHA-256").digest(token.toByteArray(Charsets.UTF_8)), + ) + + fun sameHash(left: String, right: String): Boolean = + MessageDigest.isEqual(left.toByteArray(Charsets.UTF_8), right.toByteArray(Charsets.UTF_8)) +} diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index de24cde7..013b07cd 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -48,6 +48,7 @@ import dev.typetype.server.db.tables.RecommendationOnboardingPreferencesTable import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.db.tables.ProfileAccountsTable +import dev.typetype.server.db.tables.PresenceKeysTable import dev.typetype.server.services.AdminSettingsService import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction @@ -156,6 +157,7 @@ object TestDatabase { RecommendationOnboardingPreferencesTable.deleteAll() RecommendationOnboardingStateTable.deleteAll() ProfileAccountsTable.deleteAll() + PresenceKeysTable.deleteAll() AdminSettingsService.clearCache() } } From 7593f948cab5ad14eb2eae9298f2f55fdb1864a0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 13:04:03 +0200 Subject: [PATCH 005/137] feat: expose playback presence --- .../kotlin/dev/typetype/server/Application.kt | 6 ++ .../dev/typetype/server/ApplicationRoutes.kt | 8 +- .../typetype/server/routes/PresenceRoutes.kt | 85 +++++++++++++++++++ .../server/routes/SessionActivityRoutes.kt | 10 ++- .../server/services/PresenceService.kt | 48 +++++++++++ 5 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/PresenceService.kt diff --git a/src/main/kotlin/dev/typetype/server/Application.kt b/src/main/kotlin/dev/typetype/server/Application.kt index 3e382395..8bc5297c 100644 --- a/src/main/kotlin/dev/typetype/server/Application.kt +++ b/src/main/kotlin/dev/typetype/server/Application.kt @@ -12,6 +12,8 @@ import dev.typetype.server.services.GitHubIssueService import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.OpenMojiProxyService import dev.typetype.server.services.InstanceService @@ -56,6 +58,8 @@ fun Application.module() { val gitHubIssueService = GitHubIssueService() val adminSettingsService = AdminSettingsService() val activeSessionService = ActiveSessionService(adminSettingsService) + val presenceKeyService = PresenceKeyService() + val presenceService = PresenceService(hasActiveKey = presenceKeyService::hasActiveKey) val restoreService = PipePipeBackupImporterService() val downloaderServiceUrl = System.getenv("DOWNLOADER_SERVICE_URL") ?: "http://typetype-downloader:18093" val youtubeSessionEncryptionKey = SecretConfigReader.read("YOUTUBE_SESSION_ENCRYPTION_KEY") @@ -118,6 +122,8 @@ fun Application.module() { authSessionConfig = authSessionConfig, adminSettingsService = adminSettingsService, activeSessionService = activeSessionService, + presenceKeyService = presenceKeyService, + presenceService = presenceService, downloaderGatewayService = downloaderGatewayService, gitHubIssueService = gitHubIssueService, instanceService = instanceService, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 2e52330d..a8d024cd 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -16,6 +16,7 @@ import dev.typetype.server.routes.downloaderGatewayRoutes import dev.typetype.server.routes.deArrowRoutes import dev.typetype.server.routes.internalObservabilityRoutes import dev.typetype.server.routes.oidcAuthRoutes +import dev.typetype.server.routes.presenceRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes @@ -43,6 +44,8 @@ import dev.typetype.server.services.PasswordResetService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.ProfileService import dev.typetype.server.services.ProfileAccountService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService import dev.typetype.server.services.UserAdminService import dev.typetype.server.services.YoutubeRemoteBrowserService import dev.typetype.server.portability.PortabilityEngine @@ -56,6 +59,8 @@ internal fun Application.installApplicationRoutes( authSessionConfig: AuthSessionConfig, adminSettingsService: AdminSettingsService, activeSessionService: ActiveSessionService, + presenceKeyService: PresenceKeyService, + presenceService: PresenceService, downloaderGatewayService: DownloaderGatewayService, gitHubIssueService: GitHubIssueService, instanceService: InstanceService, @@ -120,7 +125,8 @@ internal fun Application.installApplicationRoutes( adminIdentityRoutes(svc.accountIdentityService, authService) adminAllowListRoutes(authService, userAdminService, svc.adminManagedAccessService, svc.adminUserLookupService, svc.allowedChannelsService, svc.allowedPlaylistsService) adminSessionRoutes(authService, activeSessionService) - sessionActivityRoutes(authService, activeSessionService) + sessionActivityRoutes(authService, activeSessionService, presenceService) + rateLimit(USER_DATA_ZONE) { presenceRoutes(authService, presenceKeyService, presenceService) } adminBugReportRoutes(authService, svc.bugReportService, gitHubIssueService) avatarRoutes(avatarService, openMojiProxyService, svc.customAvatarService) rateLimit(USER_DATA_ZONE) { youtubeRemoteBrowserRoutes(youtubeRemoteBrowserService, authService) } diff --git a/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt new file mode 100644 index 00000000..edd8c04e --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.PresenceKeyCreateRequest +import dev.typetype.server.models.PresenceResponse +import dev.typetype.server.models.PresenceScopes +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceKeyLimitException +import dev.typetype.server.services.PresenceService +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +fun Route.presenceRoutes( + authService: dev.typetype.server.services.AuthService, + presenceKeyService: PresenceKeyService, + presenceService: PresenceService, +): Unit { + post("/presence/keys") { + call.withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Guests cannot create presence keys")) + } + val body = runCatching { call.receive() }.getOrNull() + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + val created = try { + presenceKeyService.create(userId, body.name ?: "") + } catch (error: PresenceKeyLimitException) { + return@withJwtAuth call.respond( + HttpStatusCode.Conflict, + ErrorResponse(error.message ?: "Presence key limit reached", "presence_key_limit"), + ) + } catch (error: IllegalArgumentException) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse(error.message ?: "Invalid request")) + } + call.respond(HttpStatusCode.Created, created) + } + } + get("/presence/keys") { + call.withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth call.respond(HttpStatusCode.Forbidden, ErrorResponse("Guests cannot own presence keys")) + } + call.respond(presenceKeyService.list(userId)) + } + } + delete("/presence/keys/{id}") { + call.withJwtAuth(authService) { userId -> + val id = call.parameters["id"].orEmpty() + val revoked = presenceKeyService.revoke(userId, id) + if (revoked) call.respond(HttpStatusCode.NoContent) + else call.respond(HttpStatusCode.NotFound, ErrorResponse("Presence key not found")) + } + } + get("/presence/now-playing") { + val principal = call.presencePrincipal(presenceKeyService) ?: return@get + val nowPlaying = presenceService.current(principal.userId) + call.response.headers.append(HttpHeaders.CacheControl, "no-store") + call.respond( + PresenceResponse( + active = nowPlaying != null, + nowPlaying = nowPlaying, + retryAfterMs = PresenceScopes.PRESENCE_RETRY_MS, + serverTimeMs = System.currentTimeMillis(), + ), + ) + } +} + +private suspend fun io.ktor.server.application.ApplicationCall.presencePrincipal( + presenceKeyService: PresenceKeyService, +): dev.typetype.server.services.PresenceTokenPrincipal? { + val authorization = request.headers["Authorization"] + val token = authorization?.takeIf { it.startsWith("Bearer ") }?.substringAfter("Bearer ")?.trim() + val principal = token?.let { presenceKeyService.resolve(it) } + if (principal == null) { + respond(HttpStatusCode.Unauthorized, ErrorResponse("Invalid presence token")) + } + return principal +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt index cfeaec91..11d3d3f2 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.models.SessionPlaybackStartRequest import dev.typetype.server.models.SessionPlaybackStopRequest import dev.typetype.server.services.ActiveSessionService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.PresenceService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.request.receive @@ -14,7 +15,11 @@ import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.post -fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: ActiveSessionService): Unit { +fun Route.sessionActivityRoutes( + authService: AuthService, + activeSessionService: ActiveSessionService, + presenceService: PresenceService = PresenceService(), +): Unit { post("/sessions/activity") { call.withJwtAuth(authService) { userId -> val body = runCatching { call.receive() }.getOrElse { @@ -34,6 +39,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing now playing fields")) } activeSessionService.reportPlaybackStart(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackStart(userId, body) call.respond(HttpStatusCode.NoContent) } } @@ -44,6 +50,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } activeSessionService.reportPlaybackProgress(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackProgress(userId, body) call.respond(HttpStatusCode.NoContent) } } @@ -54,6 +61,7 @@ fun Route.sessionActivityRoutes(authService: AuthService, activeSessionService: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) } activeSessionService.reportPlaybackStop(userId, body, call.request.headers[HttpHeaders.UserAgent]) + presenceService.reportPlaybackStop(userId) call.respond(HttpStatusCode.NoContent) } } diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceService.kt b/src/main/kotlin/dev/typetype/server/services/PresenceService.kt new file mode 100644 index 00000000..30cc7feb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/PresenceService.kt @@ -0,0 +1,48 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ActiveSessionNowPlayingItem +import dev.typetype.server.models.SessionPlaybackProgressRequest +import dev.typetype.server.models.SessionPlaybackStartRequest +import java.util.concurrent.ConcurrentHashMap + +class PresenceService( + private val nowProvider: () -> Long = System::currentTimeMillis, + private val hasActiveKey: suspend (String) -> Boolean = { false }, +) { + private val nowPlaying = ConcurrentHashMap() + + suspend fun reportPlaybackStart(userId: String, request: SessionPlaybackStartRequest): Unit = + record(userId, ActiveSessionNowPlayingMapper.fromStart(request, nowProvider())) + + suspend fun reportPlaybackProgress(userId: String, request: SessionPlaybackProgressRequest): Unit { + val current = nowPlaying[userId] + record(userId, ActiveSessionNowPlayingMapper.fromProgress(current, request, nowProvider())) + } + + fun reportPlaybackStop(userId: String): Unit { + nowPlaying.remove(userId) + } + + suspend fun current(userId: String): ActiveSessionNowPlayingItem? { + val now = nowProvider() + pruneExpired(now) + return nowPlaying[userId]?.takeIf { now - it.updatedAt <= ACTIVITY_TTL_MS } + } + + private suspend fun record(userId: String, value: ActiveSessionNowPlayingItem?) { + if (value == null) return + if (!hasActiveKey(userId)) { + nowPlaying.remove(userId) + return + } + nowPlaying[userId] = value + } + + private fun pruneExpired(now: Long): Unit { + nowPlaying.entries.removeIf { now - it.value.updatedAt > ACTIVITY_TTL_MS } + } + + companion object { + const val ACTIVITY_TTL_MS = 120_000L + } +} From 078e6632ee807a67fbf4fd3d87443b3cba794d6d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 13:04:03 +0200 Subject: [PATCH 006/137] test: cover presence key lifecycle --- .../dev/typetype/server/PresenceRoutesTest.kt | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt new file mode 100644 index 00000000..7dbbb5fc --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt @@ -0,0 +1,171 @@ +package dev.typetype.server + +import dev.typetype.server.routes.adminSessionRoutes +import dev.typetype.server.routes.presenceRoutes +import dev.typetype.server.routes.sessionActivityRoutes +import dev.typetype.server.services.ActiveSessionService +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.PresenceKeyService +import dev.typetype.server.services.PresenceService +import io.ktor.client.request.bearerAuth +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonPrimitive +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class PresenceRoutesTest { + private val presenceKeys = PresenceKeyService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `presence key can read now playing and is revoked independently`() = runBlocking { + insertActiveSessionUser() + val presenceService = PresenceService(hasActiveKey = presenceKeys::hasActiveKey) + withRoutes(presenceService) { + val created = client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Discord RPC"}""") + } + assertEquals(HttpStatusCode.Created, created.status) + val createdBody = Json.parseToJsonElement(created.bodyAsText()).jsonObject + val token = createdBody["token"]?.jsonPrimitive?.contentOrNull ?: error("missing token") + val keyId = createdBody["key"]?.jsonObject?.get("id")?.jsonPrimitive?.contentOrNull + assertTrue(token.startsWith("ttp1_")) + + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + }.let { assertEquals(HttpStatusCode.NoContent, it.status) } + + val presence = client.get("/presence/now-playing") { bearerAuth(token) } + assertEquals(HttpStatusCode.OK, presence.status) + val presenceBody = Json.parseToJsonElement(presence.bodyAsText()).jsonObject + assertEquals(true, presenceBody["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + assertEquals("Video", presenceBody["nowPlaying"]?.jsonObject?.get("title")?.jsonPrimitive?.contentOrNull) + assertEquals("Channel", presenceBody["nowPlaying"]?.jsonObject?.get("channelName")?.jsonPrimitive?.contentOrNull) + + val unauthorized = client.get("/presence/now-playing") { bearerAuth("test-jwt") } + assertEquals(HttpStatusCode.Unauthorized, unauthorized.status) + + val listed = Json.parseToJsonElement( + client.get("/presence/keys") { bearerAuth("test-jwt") }.bodyAsText(), + ) + assertTrue(listed.toString().contains(""""id":"$keyId"""")) + assertFalse(listed.toString().contains(""""token":"$token"""")) + + val revoked = client.delete("/presence/keys/$keyId") { bearerAuth("test-jwt") } + assertEquals(HttpStatusCode.NoContent, revoked.status) + assertEquals(HttpStatusCode.Unauthorized, client.get("/presence/now-playing") { bearerAuth(token) }.status) + } + } + + @Test + fun `presence works without admin active session tracking`() = runBlocking { + insertActiveSessionUser() + val presenceService = PresenceService(hasActiveKey = presenceKeys::hasActiveKey) + withRoutes(presenceService) { + val created = client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Local monitor"}""") + } + val token = Json.parseToJsonElement(created.bodyAsText()).jsonObject["token"] + ?.jsonPrimitive?.contentOrNull ?: error("missing token") + val adminSessions = client.get("/admin/sessions") { bearerAuth("test-jwt") } + assertEquals("[]", adminSessions.bodyAsText()) + + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + } + val presence = Json.parseToJsonElement( + client.get("/presence/now-playing") { bearerAuth(token) }.bodyAsText(), + ).jsonObject + assertEquals(true, presence["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + } + } + + @Test + fun `stale presence reports inactive`() = runBlocking { + var now = 1_000L + insertActiveSessionUser() + val keyService = PresenceKeyService { now } + val presenceService = PresenceService({ now }, keyService::hasActiveKey) + withRoutes(presenceService, keyService) { + val token = Json.parseToJsonElement( + client.post("/presence/keys") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody("""{"name":"Clock"}""") + }.bodyAsText(), + ).jsonObject["token"]?.jsonPrimitive?.contentOrNull ?: error("missing token") + client.post("/sessions/playback/start") { + bearerAuth("test-jwt") + contentType(ContentType.Application.Json) + setBody(playbackBody()) + } + now += PresenceService.ACTIVITY_TTL_MS + 1 + val presence = Json.parseToJsonElement( + client.get("/presence/now-playing") { bearerAuth(token) }.bodyAsText(), + ).jsonObject + assertEquals(false, presence["active"]?.jsonPrimitive?.content?.toBooleanStrictOrNull()) + } + } + + private fun withRoutes( + presenceService: PresenceService, + keyService: PresenceKeyService = presenceKeys, + block: suspend ApplicationTestBuilder.() -> Unit, + ): Unit = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + val auth = AuthService.fixed(TEST_USER_ID) + adminSessionRoutes(auth, ActiveSessionService(AdminSettingsService())) + presenceRoutes(auth, keyService, presenceService) + sessionActivityRoutes(auth, ActiveSessionService(AdminSettingsService()), presenceService) + } + } + block() + } + + private fun playbackBody(): String = """ + {"clientName":"web","deviceId":"device-1","videoUrl":"https://example.test/watch?v=1", + "title":"Video","channelName":"Channel","positionMs":1000,"durationMs":60000,"paused":false} + """.trimIndent() +} From cc2183b3360970ff8b9818b16c18ef193f233667 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 13:04:03 +0200 Subject: [PATCH 007/137] docs: document presence API --- openapi.yaml | 3 + openapi/paths/presence.yaml | 111 ++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 openapi/paths/presence.yaml diff --git a/openapi.yaml b/openapi.yaml index 72d2cc37..46a836d0 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -69,6 +69,9 @@ paths: /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /progress/batch: { $ref: ./openapi/paths/progress.yaml#/ProgressBatch } + /presence/keys: { $ref: ./openapi/paths/presence.yaml#/PresenceKeys } + /presence/keys/{id}: { $ref: ./openapi/paths/presence.yaml#/PresenceKeyRoute } + /presence/now-playing: { $ref: ./openapi/paths/presence.yaml#/PresenceNowPlaying } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats } diff --git a/openapi/paths/presence.yaml b/openapi/paths/presence.yaml new file mode 100644 index 00000000..a22187b2 --- /dev/null +++ b/openapi/paths/presence.yaml @@ -0,0 +1,111 @@ +PresenceKeys: + post: + tags: [user-data] + summary: Create a presence-only access key + description: The returned token is displayed once and grants only the presence:read scope. + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/PresenceKeyName' } + responses: + '201': + description: Created key metadata and the one-time token. + content: + application/json: + schema: { $ref: '#/PresenceKeyCreated' } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + get: + tags: [user-data] + summary: List presence access keys + security: [{ bearerAuth: [] }] + responses: + '200': + description: Key metadata without token material. + content: + application/json: + schema: + type: array + items: { $ref: '#/PresenceKey' } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + +PresenceKeyRoute: + delete: + tags: [user-data] + summary: Revoke a presence access key + security: [{ bearerAuth: [] }] + parameters: + - name: id + in: path + required: true + schema: { type: string, format: uuid } + responses: + '204': { description: Key revoked. } + '401': { $ref: ../components/common.yaml#/JsonError } + '403': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + +PresenceNowPlaying: + get: + tags: [user-data] + summary: Read the user's current playback presence + description: Requires a dedicated presence key. Clients normally poll every 15 to 30 seconds. + security: [{ bearerAuth: [] }] + responses: + '200': + description: Current presence, including an explicit inactive state. + content: + application/json: + schema: { $ref: '#/PresenceResponse' } + '401': { $ref: ../components/common.yaml#/JsonError } + +PresenceKeyName: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 50 } + +PresenceKeyCreated: + type: object + required: [key, token] + properties: + key: { $ref: '#/PresenceKey' } + token: { type: string, description: One-time presence token in ttp1_ form. } + +PresenceKey: + type: object + required: [id, name, tokenPrefix, scope, createdAt] + properties: + id: { type: string, format: uuid } + name: { type: string } + tokenPrefix: { type: string } + scope: { type: string, enum: [presence:read] } + createdAt: { type: integer, format: int64 } + lastUsedAt: { type: integer, format: int64, nullable: true } + +PresenceMedia: + type: object + required: [videoUrl, title, positionMs, paused, updatedAt] + properties: + videoUrl: { type: string } + title: { type: string } + thumbnail: { type: string, nullable: true } + channelName: { type: string, nullable: true } + positionMs: { type: integer, format: int64, minimum: 0 } + durationMs: { type: integer, format: int64, minimum: 0, nullable: true } + paused: { type: boolean } + updatedAt: { type: integer, format: int64 } + +PresenceResponse: + type: object + required: [active, retryAfterMs, serverTimeMs] + properties: + active: { type: boolean } + nowPlaying: { $ref: '#/PresenceMedia' } + retryAfterMs: { type: integer, format: int64, minimum: 0 } + serverTimeMs: { type: integer, format: int64 } From 604750b3ea619977f5f3eaafb69fb50f8085f171 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 17:19:47 +0200 Subject: [PATCH 008/137] feat: rotate home recommendations on refresh --- .../services/HomeRecommendationCursor.kt | 1 + .../services/HomeRecommendationCursorCodec.kt | 2 + .../HomeRecommendationCursorFactory.kt | 2 + .../HomeRecommendationCursorPayload.kt | 1 + .../services/HomeRecommendationMixState.kt | 1 + .../services/HomeRecommendationNextCursor.kt | 1 + .../services/HomeRecommendationPageBuilder.kt | 32 +++++-- .../HomeRecommendationPoolRotation.kt | 38 ++++++++ .../HomeRecommendationCursorCodecTest.kt | 2 + .../HomeRecommendationPoolRotationTest.kt | 96 +++++++++++++++++++ 10 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt create mode 100644 src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt index dc488378..4ecf90ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt @@ -5,6 +5,7 @@ data class HomeRecommendationCursor( val discoveryIndex: Int = 0, val subscriptionRun: Int = 0, val preferDiscovery: Boolean = true, + val rotationSeed: Long = 0L, val recentChannels: List = emptyList(), val recentSemanticKeys: List = emptyList(), val creatorMomentum: Map = emptyMap(), diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt index 748aba72..4a0fe7e4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt @@ -27,6 +27,7 @@ object HomeRecommendationCursorCodec { discoveryIndex = full.d, subscriptionRun = full.r, preferDiscovery = full.p == 1, + rotationSeed = full.f, recentChannels = full.c, recentSemanticKeys = full.k, creatorMomentum = full.m, @@ -75,6 +76,7 @@ object HomeRecommendationCursorCodec { d = cursor.discoveryIndex, r = cursor.subscriptionRun, p = if (cursor.preferDiscovery) 1 else 0, + f = cursor.rotationSeed, c = cursor.recentChannels, k = cursor.recentSemanticKeys, m = cursor.creatorMomentum, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt index cb017b2b..f759dcd3 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt @@ -6,6 +6,7 @@ object HomeRecommendationCursorFactory { discoveryIndex: Int, subscriptionRun: Int, preferDiscovery: Boolean, + rotationSeed: Long, personaState: HomeRecommendationPersonaState, snapshot: HomeRecommendationCursorMemory, ): String = HomeRecommendationCursorCodec.encode( @@ -14,6 +15,7 @@ object HomeRecommendationCursorFactory { discoveryIndex = discoveryIndex, subscriptionRun = subscriptionRun, preferDiscovery = preferDiscovery, + rotationSeed = rotationSeed, recentChannels = snapshot.recentChannels, recentSemanticKeys = snapshot.recentSemanticKeys, creatorMomentum = snapshot.creatorMomentum, diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt index fd7a4d05..2275d6f4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt @@ -8,6 +8,7 @@ data class HomeRecommendationCursorPayload( val d: Int, val r: Int, val p: Int, + val f: Long = 0L, val c: List = emptyList(), val k: List = emptyList(), val m: Map = emptyMap(), diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt index 6dbbc459..910ccc2a 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem class HomeRecommendationMixState(cursor: HomeRecommendationCursor, context: HomeRecommendationSessionContext) { + val rotationSeed = cursor.rotationSeed val channelCount = mutableMapOf() val memory = HomeRecommendationMomentumMemory(cursor) var personaState = HomeRecommendationPersonaDrift.seed(context, cursor.personaState) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt index b96a9ee8..b85bc1fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt @@ -17,6 +17,7 @@ object HomeRecommendationNextCursor { discoveryIndex = state.discoveryIndex, subscriptionRun = state.subscriptionRun, preferDiscovery = state.preferDiscovery, + rotationSeed = state.rotationSeed, personaState = state.personaState, snapshot = snapshot, ) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt index 1e1a1483..62bf3af4 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt @@ -8,20 +8,30 @@ object HomeRecommendationPageBuilder { mode: HomeRecommendationPoolMode, poolResolver: HomeRecommendationPoolResolver, ): HomeRecommendationsResponse { - val pool = poolResolver.resolve( + val cursor = if (mode == HomeRecommendationPoolMode.FULL) { + args.cursor.withRotationSeed() + } else { + args.cursor + } + val resolvedPool = poolResolver.resolve( userId = args.userId, serviceId = args.serviceId, mode = mode, context = args.context, ) + val pool = if (mode == HomeRecommendationPoolMode.FULL && cursor.rotationSeed != 0L) { + HomeRecommendationPoolRotation.apply(resolvedPool, cursor.rotationSeed) + } else { + resolvedPool + } val page = HomeRecommendationMixer.mix( pool = pool, - cursor = args.cursor, + cursor = cursor, limit = args.limit, context = args.context.sessionContext, sourceWeights = HomeRecommendationExploreBonus.apply( sourceWeights = pool.sourceWeights, - pageIndex = HomeRecommendationCursorPageIndex.from(args.cursor, args.limit), + pageIndex = HomeRecommendationCursorPageIndex.from(cursor, args.limit), ), mode = mode, userId = args.userId, @@ -31,19 +41,19 @@ object HomeRecommendationPageBuilder { val refresh = HomeRecommendationShortsRefresher.refresh( pool = pool, page = page, - cursor = args.cursor, + cursor = cursor, ) if (refresh.pool == pool && refresh.cursorOverride == null) { page } else { HomeRecommendationMixer.mix( pool = refresh.pool, - cursor = refresh.cursorOverride ?: args.cursor, + cursor = refresh.cursorOverride ?: cursor, limit = args.limit, context = args.context.sessionContext, sourceWeights = HomeRecommendationExploreBonus.apply( sourceWeights = refresh.pool.sourceWeights, - pageIndex = HomeRecommendationCursorPageIndex.from(refresh.cursorOverride ?: args.cursor, args.limit), + pageIndex = HomeRecommendationCursorPageIndex.from(refresh.cursorOverride ?: cursor, args.limit), ), mode = mode, userId = args.userId, @@ -64,4 +74,14 @@ object HomeRecommendationPageBuilder { }, ) } + + private fun HomeRecommendationCursor.withRotationSeed(): HomeRecommendationCursor = + if (rotationSeed != 0L || !isInitialPage()) { + this + } else { + copy(rotationSeed = HomeRecommendationPoolRotation.newSeed()) + } + + private fun HomeRecommendationCursor.isInitialPage(): Boolean = + subscriptionIndex == 0 && discoveryIndex == 0 && subscriptionRun == 0 && recentUrls.isEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt new file mode 100644 index 00000000..bb11c2b1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt @@ -0,0 +1,38 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.HomeRecommendationPool +import dev.typetype.server.models.VideoItem +import java.util.Random +import java.util.concurrent.ThreadLocalRandom + +object HomeRecommendationPoolRotation { + private const val ROTATION_WINDOW_SIZE = 40 + private const val SUBSCRIPTION_SALT = 0x5EED1234L + private const val DISCOVERY_SALT = 0xD15C0A7L + + fun newSeed(): Long { + var seed = ThreadLocalRandom.current().nextLong() + while (seed == 0L) seed = ThreadLocalRandom.current().nextLong() + return seed + } + + fun apply(pool: HomeRecommendationPool, seed: Long): HomeRecommendationPool = pool.copy( + subscriptions = rotate(pool.subscriptions, seed xor SUBSCRIPTION_SALT), + discovery = rotate(pool.discovery, seed xor DISCOVERY_SALT), + ) + + private fun rotate(source: List, seed: Long): List { + if (source.size <= 1) return source + val windowSize = minOf(source.size, ROTATION_WINDOW_SIZE) + if (windowSize <= 1) return source + val window = source.take(windowSize).toMutableList() + val random = Random(seed) + for (index in window.lastIndex downTo 1) { + val swapIndex = random.nextInt(index + 1) + val item = window[index] + window[index] = window[swapIndex] + window[swapIndex] = item + } + return window + source.drop(windowSize) + } +} diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt index 3a88482a..e94dd418 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt @@ -24,6 +24,7 @@ class HomeRecommendationCursorCodecTest { discoveryIndex = 12, subscriptionRun = 2, preferDiscovery = false, + rotationSeed = 123456789L, recentChannels = listOf("c1", "c2"), recentSemanticKeys = listOf("linux|kernel"), creatorMomentum = mapOf("https://yt.com/c/a" to 2), @@ -38,6 +39,7 @@ class HomeRecommendationCursorCodecTest { assertEquals(12, decoded?.discoveryIndex) assertEquals(2, decoded?.subscriptionRun) assertEquals(false, decoded?.preferDiscovery) + assertEquals(123456789L, decoded?.rotationSeed) assertEquals(listOf("c1", "c2"), decoded?.recentChannels) assertEquals(listOf("linux|kernel"), decoded?.recentSemanticKeys) assertEquals(2, decoded?.creatorMomentum?.get("https://yt.com/c/a")) diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt new file mode 100644 index 00000000..52746c5c --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server + +import dev.typetype.server.models.HomeRecommendationPool +import dev.typetype.server.services.HomeRecommendationApiArgs +import dev.typetype.server.services.HomeRecommendationContext +import dev.typetype.server.services.HomeRecommendationCursor +import dev.typetype.server.services.HomeRecommendationCursorCodec +import dev.typetype.server.services.HomeRecommendationMixer +import dev.typetype.server.services.HomeRecommendationPageBuilder +import dev.typetype.server.services.HomeRecommendationPoolMode +import dev.typetype.server.services.HomeRecommendationPoolResolver +import dev.typetype.server.services.HomeRecommendationPoolRotation +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test + +class HomeRecommendationPoolRotationTest { + @Test + fun `same seed keeps the pool order stable`() { + val pool = pool() + + val first = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + val second = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + + assertEquals(first, second) + } + + @Test + fun `different seeds rotate the candidate window`() { + val pool = pool() + + val first = HomeRecommendationPoolRotation.apply(pool, seed = 11L) + val second = HomeRecommendationPoolRotation.apply(pool, seed = 12L) + + assertNotEquals(first.subscriptions, second.subscriptions) + assertNotEquals(first.discovery, second.discovery) + assertEquals(pool.subscriptions.map { it.url }.toSet(), first.subscriptions.map { it.url }.toSet()) + assertEquals(pool.discovery.map { it.url }.toSet(), first.discovery.map { it.url }.toSet()) + assertEquals(pool.subscriptions.drop(40), first.subscriptions.drop(40)) + assertEquals(pool.discovery.drop(40), first.discovery.drop(40)) + } + + @Test + fun `the rotation seed stays with the cursor across pages`() { + val pool = HomeRecommendationPoolRotation.apply(pool(), seed = 11L) + val first = HomeRecommendationMixer.mix( + pool = pool, + cursor = HomeRecommendationCursor(rotationSeed = 11L), + limit = 20, + context = HomeRecommendationItemFixtures.context, + ) + val nextCursor = HomeRecommendationCursorCodec.decode(first.nextCursor) + val second = HomeRecommendationMixer.mix( + pool = pool, + cursor = nextCursor ?: error("expected a next cursor"), + limit = 20, + context = HomeRecommendationItemFixtures.context, + ) + + assertEquals(11L, nextCursor.rotationSeed) + assertEquals(emptySet(), first.items.map { it.url }.toSet() intersect second.items.map { it.url }.toSet()) + } + + @Test + fun `initial home pages receive a fresh seed while a seeded page is stable`() = runTest { + val resolver = mockk() + coEvery { resolver.resolve(any(), any(), any(), any()) } returns pool() + val args = HomeRecommendationApiArgs( + userId = "user", + serviceId = 0, + limit = 20, + cursor = HomeRecommendationCursor(), + context = HomeRecommendationContext(serviceId = 0, sessionContext = HomeRecommendationItemFixtures.context), + ) + + val first = HomeRecommendationPageBuilder.build(args, HomeRecommendationPoolMode.FULL, resolver) + val second = HomeRecommendationPageBuilder.build(args, HomeRecommendationPoolMode.FULL, resolver) + val firstCursor = HomeRecommendationCursorCodec.decode(first.nextCursor) + + assertNotEquals(first.items, second.items) + assertNotEquals(0L, firstCursor?.rotationSeed) + + val seededArgs = args.copy(cursor = HomeRecommendationCursor(rotationSeed = 11L)) + val seededFirst = HomeRecommendationPageBuilder.build(seededArgs, HomeRecommendationPoolMode.FULL, resolver) + val seededSecond = HomeRecommendationPageBuilder.build(seededArgs, HomeRecommendationPoolMode.FULL, resolver) + assertEquals(seededFirst.items, seededSecond.items) + } + + private fun pool(): HomeRecommendationPool = HomeRecommendationPool( + subscriptions = (1..50).map { index -> HomeRecommendationItemFixtures.video("s$index", "s$index") }, + discovery = (1..50).map { index -> HomeRecommendationItemFixtures.video("d$index", "d$index") }, + ) +} From 65d58d3dd4bdb814e5818e8e4b0f2f1a20508ac7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 19:12:34 +0200 Subject: [PATCH 009/137] fix: clarify provider extraction errors --- openapi/components/common.yaml | 9 ++++++- .../server/routes/AudioOnlyHeadResponse.kt | 4 +-- .../server/routes/AudioOnlyResponseMapper.kt | 6 ++--- .../typetype/server/routes/AudioOnlyRoutes.kt | 12 ++++----- .../typetype/server/routes/ManifestRoutes.kt | 12 ++++----- .../server/routes/ProxyResponseWriter.kt | 4 +-- .../server/routes/SabrManifestHandler.kt | 8 +++--- .../routes/SabrSessionDescriptorHandler.kt | 8 +++--- .../typetype/server/routes/StreamRoutes.kt | 5 +++- .../services/AudioOnlyStreamResolver.kt | 7 ++++-- .../server/services/HlsManifestService.kt | 19 +++++++++----- .../server/services/ManifestService.kt | 6 ++--- .../server/services/NativeManifestService.kt | 2 +- .../services/StreamExtractionErrorMapper.kt | 25 +++++++++++++++++-- .../dev/typetype/server/ManifestRoutesTest.kt | 1 + .../typetype/server/ManifestServiceTest.kt | 1 + .../server/StreamExtractionErrorMapperTest.kt | 24 +++++++++++++++++- 17 files changed, 109 insertions(+), 44 deletions(-) diff --git a/openapi/components/common.yaml b/openapi/components/common.yaml index d40be441..5294ca04 100644 --- a/openapi/components/common.yaml +++ b/openapi/components/common.yaml @@ -8,7 +8,14 @@ ErrorResponse: required: [error] properties: error: { type: string } - code: { type: string, default: error } + code: + type: string + default: error + description: >- + Stable machine-readable error code when available. Playback extraction + routes may return geographic_restriction, provider_access_blocked, + private_content, no_playable_streams, age_restricted, scheduled_premiere, + paid_content, members_only, or youtube_session_required. requestId: { type: string, nullable: true } RequestIdHeader: description: Request correlation id returned by TypeType-Server. diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt index 44f77935..936f2f8b 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt @@ -38,7 +38,7 @@ internal suspend fun ApplicationCall.respondAudioOnlyHead( respondOutputStream(containerMime(result.data.stream.mimeType), HttpStatusCode.OK, length ?: 0L) {} } } - is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt index 6162efd2..6e4040e8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt @@ -21,11 +21,11 @@ internal fun AudioOnlyStreamSelection.toResponse( val src = when (kind) { AudioOnlyStreamKind.Progressive -> "/streams/audio-only/source?token=${encode(token)}" AudioOnlyStreamKind.Hls -> hlsTokenService?.createPath(stream.url) - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") AudioOnlyStreamKind.Dash -> stream.manifestUrl?.let { audioOnlyDashManifest(it) } - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") AudioOnlyStreamKind.SabrHls -> stream.manifestUrl?.let { audioOnlySabrHlsManifest(it, token) } - ?: return ExtractionResult.Failure("No audio-only stream is available") + ?: return ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") } return ExtractionResult.Success(AudioOnlyStreamResponse( src = src, diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt index 761e5f02..7ee873c8 100644 --- a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt @@ -61,12 +61,12 @@ fun Route.audioOnlyContractRoutes( preferredLocale, )) { is ExtractionResult.Success -> call.respond(response.data) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(response.message)) - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(response.message)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(response.message, response.code)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(response.message, response.code)) } } - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } } @@ -99,8 +99,8 @@ internal fun Route.audioOnlySourceRoutes( .ensureProgressiveAudio(), contentTypeOverride = result.data.stream.mimeType, ) - is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } } diff --git a/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt index 29222c84..f5aa5077 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt @@ -38,9 +38,9 @@ fun Route.manifestRoutes( is ExtractionResult.Success -> call.respondText(result.data, ContentType.parse("application/dash+xml")) is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } @@ -53,9 +53,9 @@ fun Route.manifestRoutes( is ExtractionResult.Success -> call.respondText(result.data, ContentType.parse("application/dash+xml")) is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } @@ -105,9 +105,9 @@ private suspend fun ApplicationCall.respondHlsResult( is ExtractionResult.Success -> respondText(result.data, ContentType.parse("application/vnd.apple.mpegurl")) is ExtractionResult.BadRequest -> - respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) is ExtractionResult.Failure -> - respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } } diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt b/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt index 4f5676a2..a82a3585 100644 --- a/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt +++ b/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt @@ -16,8 +16,8 @@ internal suspend fun ApplicationCall.respondProxyResult( contentTypeOverride: String? = null, ): Unit = when (result) { is ExtractionResult.Success -> respondProxy(result.data, contentTypeOverride) - is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) - is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) + is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) } private suspend fun ApplicationCall.respondProxy(proxy: ProxyResponse, contentTypeOverride: String?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt index b98707ce..6f4fce76 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt @@ -69,9 +69,9 @@ internal class SabrManifestHandler( } } is ExtractionResult.Failure -> - return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) + return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) is ExtractionResult.BadRequest -> - return call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) + return call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) } } val requestedStartTimeMs = call.request.queryParameters["playerTimeMs"]?.toLongOrNull()?.coerceAtLeast(0L) @@ -93,7 +93,7 @@ internal class SabrManifestHandler( ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR audio for this video"), + ErrorResponse("No SABR audio for this video", "no_playable_streams"), ) val video = SabrFormatSelector.video( prepared.info, @@ -101,7 +101,7 @@ internal class SabrManifestHandler( ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR video for this video"), + ErrorResponse("No SABR video for this video", "no_playable_streams"), ) val startTimeMs = downloadRange?.startTimeMs(audio, video, audioOnly) ?: requestedStartTimeMs val userId = audioToken?.userId ?: access?.userId ?: videoId diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt b/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt index acd908d5..97b9c92a 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt @@ -39,11 +39,11 @@ internal class SabrSessionDescriptorHandler( } is ExtractionResult.Failure -> return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse(result.message), + ErrorResponse(result.message, result.code), ) is ExtractionResult.BadRequest -> return call.respond( HttpStatusCode.BadRequest, - ErrorResponse(result.message), + ErrorResponse(result.message, result.code), ) } } @@ -59,7 +59,7 @@ internal class SabrSessionDescriptorHandler( requireAac = true, ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR audio for this video"), + ErrorResponse("No SABR audio for this video", "no_playable_streams"), ) val video = SabrFormatSelector.video( prepared.info, @@ -67,7 +67,7 @@ internal class SabrSessionDescriptorHandler( ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No SABR video for this video"), + ErrorResponse("No SABR video for this video", "no_playable_streams"), ) val userId = access.userId ?: videoId val holder = createHolder(videoId, userId, prepared, audio, video, startTimeMs) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index cc1070f3..bd033d15 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -131,7 +131,10 @@ private fun Route.streamRoute( if (!data.hasPlayableSource()) { return@get call.respond( HttpStatusCode.UnprocessableEntity, - ErrorResponse("No playable streams available", "no_playable_streams"), + ErrorResponse( + "No compatible stream is available for this video", + "no_playable_streams", + ), ) } val publicData = try { diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt b/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt index d0f1f103..37235e79 100644 --- a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt +++ b/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt @@ -62,10 +62,10 @@ class AudioOnlyStreamResolver( if (sabr != null) return ExtractionResult.Success( AudioOnlyStreamSelection(response, sabr, AudioOnlyStreamKind.SabrHls) ) - if (selectedItag != null) return ExtractionResult.Failure("No audio-only stream is available") + if (selectedItag != null) return noAudioStream() val hls = (if (allowHls) hlsCandidate(response, preferOriginal, preferredLocale) else null) if (hls != null) return ExtractionResult.Success(AudioOnlyStreamSelection(response, hls, AudioOnlyStreamKind.Hls)) - return ExtractionResult.Failure("No audio-only stream is available") + return noAudioStream() } private fun hlsCandidate(response: StreamResponse, preferOriginal: Boolean, preferredLocale: String?) = @@ -78,6 +78,9 @@ class AudioOnlyStreamResolver( private fun dev.typetype.server.models.AudioStreamItem.matchesSelected(itag: Int, trackId: String?): Boolean = this.itag == itag && this.audioTrackId == trackId + private fun noAudioStream(): ExtractionResult.Failure = + ExtractionResult.Failure("No audio-only stream is available", "no_playable_streams") + private fun ExtractionResult?.resolveWith( sessionResult: ExtractionResult?, ): ExtractionResult { diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index 5a07b108..e1f13d23 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -82,13 +82,20 @@ class HlsManifestService( result: ExtractionResult, allowAttestedYoutubeHls: Boolean = false, ): ExtractionResult { - if (result is ExtractionResult.BadRequest) return result - if (result !is ExtractionResult.Success) return ExtractionResult.Failure("No HLS stream available for this video") - if (allowAttestedYoutubeHls && result.data.isLive) { - attestedYoutubeHls(result.data.id)?.let { return ExtractionResult.Success(it) } + val success = when (result) { + is ExtractionResult.BadRequest -> return result + is ExtractionResult.Failure -> return ExtractionResult.Failure(result.message, result.code, result.kind) + is ExtractionResult.Success -> result + } + if (allowAttestedYoutubeHls && success.data.isLive) { + attestedYoutubeHls(success.data.id)?.let { return ExtractionResult.Success(it) } + } + val hls = success.data.hlsUrl + return if (hls.isNotBlank()) { + ExtractionResult.Success(hls) + } else { + ExtractionResult.Failure("No HLS stream available for this video", "no_playable_streams") } - val hls = result.data.hlsUrl - return if (hls.isNotBlank()) ExtractionResult.Success(hls) else ExtractionResult.Failure("No HLS stream available for this video") } private suspend fun fetchAndRewrite(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult = diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt index 81c2ed36..c4ac4583 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/ManifestService.kt @@ -17,7 +17,7 @@ class ManifestService( val videos = compatibleVideoStreams(info.videoOnlyStreams) val audios = compatibleAudioStreams(info.audioStreams, info.preferredDefaultAudioTrackId) if (videos.isEmpty() && audios.isEmpty()) - return ExtractionResult.Failure("No compatible streams found for DASH manifest") + return ExtractionResult.Failure("No compatible streams found for DASH manifest", "no_playable_streams") return ExtractionResult.Success(buildMpd(videos, audios, info.duration)) } @@ -129,7 +129,7 @@ class ManifestService( URLEncoder.encode(url, StandardCharsets.UTF_8) private fun ExtractionResult.recast(): ExtractionResult = when (this) { is ExtractionResult.Success -> ExtractionResult.Success(data.toString()) - is ExtractionResult.BadRequest -> ExtractionResult.BadRequest(message) - is ExtractionResult.Failure -> ExtractionResult.Failure(message) + is ExtractionResult.BadRequest -> ExtractionResult.BadRequest(message, code) + is ExtractionResult.Failure -> ExtractionResult.Failure(message, code, kind) } } diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt b/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt index 0d216102..bd8c57c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt @@ -40,7 +40,7 @@ class NativeManifestService { val audios = compatibleAudioStreams(info.audioStreams) val preferredAudioTrackId = resolvePreferredAudioTrackId(audios) if (videos.isEmpty() && audios.isEmpty()) - return ExtractionResult.Failure("No compatible streams found") + return ExtractionResult.Failure("No compatible streams found", "no_playable_streams") return runCatching { ExtractionResult.Success( NativeManifestBuilder.build(videos, audios, info.duration, preferredAudioTrackId) diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 2be09e73..62b2cfdb 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -3,16 +3,26 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ExtractionFailureKind import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException +import org.schabi.newpipe.extractor.exceptions.AntiBotException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException internal object StreamExtractionErrorMapper { const val MEMBERS_ONLY_FALLBACK = "This video is only available for members" const val PAID_CONTENT_FALLBACK = "This video is a paid video" + const val GEOGRAPHIC_RESTRICTION_CODE = "geographic_restriction" + const val PROVIDER_ACCESS_BLOCKED_CODE = "provider_access_blocked" + const val PRIVATE_CONTENT_CODE = "private_content" + const val GEOGRAPHIC_RESTRICTION_FALLBACK = + "This video is not available in the server's region. A VPN or another outbound network may help." + const val PROVIDER_ACCESS_BLOCKED_FALLBACK = + "The video provider is blocking requests from this TypeType server. Try a VPN or another outbound network for the server, then retry." + const val PRIVATE_CONTENT_FALLBACK = "This video is private or no longer available" fun map(error: Throwable, sourceUrl: String? = null, fallback: String = "Extraction failed"): ExtractionResult = mapByType(error, fallback) @@ -35,8 +45,19 @@ internal object StreamExtractionErrorMapper { sanitize(error.message) ?: "This video is age-restricted", "age_restricted", ) - is GeographicRestrictionException, - is PrivateContentException -> ExtractionResult.BadRequest(sanitize(error.message) ?: "Content not available") + is GeographicRestrictionException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: GEOGRAPHIC_RESTRICTION_FALLBACK, + GEOGRAPHIC_RESTRICTION_CODE, + ) + is PrivateContentException -> ExtractionResult.BadRequest( + sanitize(error.message) ?: PRIVATE_CONTENT_FALLBACK, + PRIVATE_CONTENT_CODE, + ) + is AntiBotException, + is ReCaptchaException -> ExtractionResult.Failure( + PROVIDER_ACCESS_BLOCKED_FALLBACK, + PROVIDER_ACCESS_BLOCKED_CODE, + ) else -> ExtractionResult.Failure( sanitize(error.message) ?: fallback, kind = if (error.isYoutubeSessionRejected()) { diff --git a/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt index ef31b29a..1a623853 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt @@ -69,6 +69,7 @@ class ManifestRoutesTest { } val response = client.get("/streams/manifest?url=https://youtube.com/watch?v=empty") assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } @Test diff --git a/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt b/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt index 298da50b..db07b3f2 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt @@ -97,6 +97,7 @@ class ManifestServiceTest { val result = service.dashManifest("https://youtube.com/watch?v=test") assertTrue(result is ExtractionResult.Failure) + assertEquals("no_playable_streams", (result as ExtractionResult.Failure).code) } @Test diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index 25db135c..ad282d34 100644 --- a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -8,9 +8,12 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException @@ -68,7 +71,26 @@ class StreamExtractionErrorMapperTest { @Test fun `maps content restrictions to bad request with extractor message`() { val result = StreamExtractionErrorMapper.map(PrivateContentException("private video")) - assertEquals(ExtractionResult.BadRequest("private video"), result) + assertEquals(ExtractionResult.BadRequest("private video", "private_content"), result) + } + + @Test + fun `maps geographic restrictions to a stable access code`() { + val result = StreamExtractionErrorMapper.map(GeographicRestrictionException("Only available in Japan")) + assertEquals(ExtractionResult.BadRequest("Only available in Japan", "geographic_restriction"), result) + } + + @Test + fun `maps provider bot blocks to a vpn-aware failure`() { + val antiBot = StreamExtractionErrorMapper.map(AntiBotException("YouTube requested CAPTCHA verification")) + val captcha = StreamExtractionErrorMapper.map(ReCaptchaException("reCaptcha requested", "https://youtube.com")) + + val expected = ExtractionResult.Failure( + StreamExtractionErrorMapper.PROVIDER_ACCESS_BLOCKED_FALLBACK, + "provider_access_blocked", + ) + assertEquals(expected, antiBot) + assertEquals(expected, captcha) } @Test From 21fd3308171efb44c7e0447ca37dd2b0048a7ff9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 16 Sep 2026 20:13:33 +0200 Subject: [PATCH 010/137] fix: batch portability history imports --- .../server/portability/PortabilityDataPort.kt | 1 + .../server/portability/PortabilityEngine.kt | 16 ++- .../TypeTypePortabilityCoreImport.kt | 128 ++++++++---------- .../TypeTypePortabilityDataPort.kt | 5 +- .../TypeTypePortabilityFilterImport.kt | 2 + .../TypeTypePortabilityHistoryImport.kt | 64 +++++++++ .../portability/TypeTypePortabilityImport.kt | 11 +- .../TypeTypePortabilityLibraryImport.kt | 2 + .../TypeTypePortabilitySettingsImport.kt | 10 +- .../PortabilityArchiveReproductionTest.kt | 1 + .../typetype/server/PortabilityRoutesTest.kt | 7 +- .../portability/PortabilityEngineTest.kt | 10 +- .../TypeTypePortabilityHistoryImportTest.kt | 85 ++++++++++++ 13 files changed, 252 insertions(+), 90 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt index 0757eb72..065c1fd3 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt @@ -6,6 +6,7 @@ interface PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit = { _, _ -> }, + onCategoryProgress: (PortabilityCategory, Long) -> Unit = { _, _ -> }, ): Map suspend fun export( diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index 9d4d83a2..4f788a5e 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -124,14 +124,22 @@ class PortabilityEngine internal constructor( private suspend fun apply(job: PortabilityJob, request: PortabilityImportRequest) { runJob(job, null) { + val source = requireNotNull(job.spool) + val counts = source.counts() + val total = request.categories.sumOf { counts[it] ?: 0L } val progress = PortabilityProgressReporter( job, PortabilityProgressPhase.APPLYING, - PortabilityProgressUnit.CATEGORIES, - request.categories.size.toLong(), - interval = 1L, + PortabilityProgressUnit.RECORDS, + total, + ) + val result = dataPort.import( + job.ownerId, + source, + request, + onCategoryProgress = { _, count -> progress.add(count) }, + onCategoryComplete = { _, _ -> }, ) - val result = dataPort.import(job.ownerId, requireNotNull(job.spool), request) { _, _ -> progress.add() } progress.finish() job.transition(setOf(PortabilityJobState.APPLYING), PortabilityJobState.COMPLETED, result = result) } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt index 5f867ea3..c93c5d11 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt @@ -1,6 +1,5 @@ package dev.typetype.server.portability -import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable @@ -23,11 +22,11 @@ internal object TypeTypePortabilityCoreImport { category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long = when (category) { - PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy) - PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy) - PortabilityCategory.HISTORY -> history(userId, source, policy) - PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy) + PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, source, policy, onRecord) + PortabilityCategory.SUBSCRIPTION_GROUPS -> groups(userId, source, policy, onRecord) + PortabilityCategory.PLAYLISTS -> playlists(userId, source, policy, onRecord) else -> error("Unsupported core portability category") } @@ -35,6 +34,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { SubscriptionMutationLock.acquire(userId) if (policy == PortabilityDuplicatePolicy.REPLACE) { @@ -42,15 +42,17 @@ internal object TypeTypePortabilityCoreImport { } var count = 0L source.forEach(PortabilityCategory.SUBSCRIPTIONS) { record -> - if (record !is PortabilitySubscription) return@forEach - val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) - count += SubscriptionsTable.insertIgnore { - it[SubscriptionsTable.userId] = userId - it[SubscriptionsTable.channelUrl] = channelUrl - it[SubscriptionsTable.name] = record.name - it[SubscriptionsTable.avatarUrl] = record.avatarUrl - it[SubscriptionsTable.subscribedAt] = record.subscribedAt - }.insertedCount + if (record is PortabilitySubscription) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + count += SubscriptionsTable.insertIgnore { + it[SubscriptionsTable.userId] = userId + it[SubscriptionsTable.channelUrl] = channelUrl + it[SubscriptionsTable.name] = record.name + it[SubscriptionsTable.avatarUrl] = record.avatarUrl + it[SubscriptionsTable.subscribedAt] = record.subscribedAt + }.insertedCount + } + onRecord() } return count } @@ -59,6 +61,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { SubscriptionMutationLock.acquire(userId) if (policy == PortabilityDuplicatePolicy.REPLACE) { @@ -67,69 +70,43 @@ internal object TypeTypePortabilityCoreImport { } var count = 0L source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> - if (record !is PortabilitySubscriptionGroup) return@forEach - val normalized = record.name.trim().lowercase(Locale.ROOT) - require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } - count += SubscriptionGroupsTable.insertIgnore { - it[id] = stableId(userId, "group:$normalized") - it[SubscriptionGroupsTable.userId] = userId - it[name] = record.name.trim() - it[normalizedName] = normalized - it[createdAt] = 0L - it[updatedAt] = 0L - }.insertedCount + if (record is PortabilitySubscriptionGroup) { + val normalized = record.name.trim().lowercase(Locale.ROOT) + require(normalized.isNotBlank() && normalized.length <= 100) { "Invalid subscription group name" } + count += SubscriptionGroupsTable.insertIgnore { + it[id] = stableId(userId, "group:$normalized") + it[SubscriptionGroupsTable.userId] = userId + it[name] = record.name.trim() + it[normalizedName] = normalized + it[createdAt] = 0L + it[updatedAt] = 0L + }.insertedCount + } + onRecord() } source.forEach(PortabilityCategory.SUBSCRIPTION_GROUPS) { record -> - if (record !is PortabilitySubscriptionGroupMembership) return@forEach - val normalized = record.groupName.trim().lowercase(Locale.ROOT) - val groupId = SubscriptionGroupsTable.selectAll().where { - (SubscriptionGroupsTable.userId eq userId) and - (SubscriptionGroupsTable.normalizedName eq normalized) - }.singleOrNull()?.get(SubscriptionGroupsTable.id) ?: return@forEach - val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) - val subscribed = SubscriptionsTable.selectAll().where { - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) - }.empty().not() - if (!subscribed) return@forEach - count += SubscriptionGroupMembershipsTable.insertIgnore { - it[SubscriptionGroupMembershipsTable.groupId] = groupId - it[SubscriptionGroupMembershipsTable.userId] = userId - it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl - it[addedAt] = 0L - }.insertedCount - } - return count - } - - private fun history( - userId: String, - source: PortabilityRecordSource, - policy: PortabilityDuplicatePolicy, - ): Long { - if (policy == PortabilityDuplicatePolicy.REPLACE) HistoryTable.deleteWhere { HistoryTable.userId eq userId } - var count = 0L - source.forEach(PortabilityCategory.HISTORY) { record -> - if (record !is PortabilityHistory) return@forEach - val exists = HistoryTable.selectAll().where { - (HistoryTable.userId eq userId) and - (HistoryTable.url eq record.video.url) and - (HistoryTable.watchedAt eq record.watchedAt) - }.empty().not() - if (!exists) { - count += HistoryTable.insertIgnore { - it[id] = UUID.randomUUID().toString() - it[HistoryTable.userId] = userId - it[url] = record.video.url - it[title] = record.video.title - it[thumbnail] = record.video.thumbnailUrl - it[channelName] = record.video.channelName - it[channelUrl] = record.video.channelUrl - it[channelAvatar] = record.video.channelAvatarUrl - it[duration] = record.video.durationSeconds - it[progress] = record.positionSeconds - it[watchedAt] = record.watchedAt - }.insertedCount + if (record is PortabilitySubscriptionGroupMembership) { + val normalized = record.groupName.trim().lowercase(Locale.ROOT) + val groupId = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalized) + }.singleOrNull()?.get(SubscriptionGroupsTable.id) + if (groupId != null) { + val channelUrl = ChannelUrlCanonicalizer.canonicalize(record.channelUrl) + val subscribed = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.empty().not() + if (subscribed) { + count += SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = 0L + }.insertedCount + } + } } + onRecord() } return count } @@ -138,6 +115,7 @@ internal object TypeTypePortabilityCoreImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { if (policy == PortabilityDuplicatePolicy.REPLACE) { PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } @@ -146,9 +124,11 @@ internal object TypeTypePortabilityCoreImport { var count = 0L source.forEach(PortabilityCategory.PLAYLISTS) { record -> if (record is PortabilityPlaylist) count += insertPlaylist(userId, record) + onRecord() } source.forEach(PortabilityCategory.PLAYLISTS) { record -> if (record is PortabilityPlaylistVideo) count += insertPlaylistVideo(userId, record) + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt index 343a103b..62bdb14c 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt @@ -9,11 +9,14 @@ class TypeTypePortabilityDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map { val result = linkedMapOf() request.categories.sortedBy(PortabilityCategory::wireName).forEach { category -> val imported = DatabaseFactory.query { - TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) + TypeTypePortabilityImport.write(userId, category, source, request.duplicatePolicy) { + onCategoryProgress(category, 1L) + } } result[category.wireName] = imported onCategoryComplete(category, imported) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt index 819dcc67..ffd07b1d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt @@ -22,11 +22,13 @@ internal object TypeTypePortabilityFilterImport { userId: String, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { if (policy == PortabilityDuplicatePolicy.REPLACE) clear(userId) var count = 0L source.forEach(PortabilityCategory.CONTENT_FILTERS) { record -> if (record is PortabilityContentFilter) count += insert(userId, record) + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt new file mode 100644 index 00000000..ca92e360 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt @@ -0,0 +1,64 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.HistoryTable +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.select +import java.util.UUID + +internal object TypeTypePortabilityHistoryImport { + private const val BATCH_SIZE = 500 + + fun write( + userId: String, + source: PortabilityRecordSource, + policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, + ): Long { + if (policy == PortabilityDuplicatePolicy.REPLACE) { + HistoryTable.deleteWhere { HistoryTable.userId eq userId } + } + val knownKeys = existingKeys(userId, policy) + val pending = ArrayList(BATCH_SIZE) + var count = 0L + source.forEach(PortabilityCategory.HISTORY) { record -> + if (record is PortabilityHistory && knownKeys.add(HistoryKey(record.video.url, record.watchedAt))) { + pending += record + } + onRecord() + if (pending.size >= BATCH_SIZE) { + count += insertBatch(userId, pending) + pending.clear() + } + } + return count + insertBatch(userId, pending) + } + + private fun existingKeys(userId: String, policy: PortabilityDuplicatePolicy): MutableSet { + if (policy == PortabilityDuplicatePolicy.REPLACE) return HashSet() + return HistoryTable.select(HistoryTable.url, HistoryTable.watchedAt) + .where { HistoryTable.userId eq userId } + .mapTo(HashSet()) { HistoryKey(it[HistoryTable.url], it[HistoryTable.watchedAt]) } + } + + private fun insertBatch(userId: String, records: List): Long { + if (records.isEmpty()) return 0L + HistoryTable.batchInsert(records, shouldReturnGeneratedValues = false) { record -> + this[HistoryTable.id] = UUID.randomUUID().toString() + this[HistoryTable.userId] = userId + this[HistoryTable.url] = record.video.url + this[HistoryTable.title] = record.video.title + this[HistoryTable.thumbnail] = record.video.thumbnailUrl + this[HistoryTable.channelName] = record.video.channelName + this[HistoryTable.channelUrl] = record.video.channelUrl + this[HistoryTable.channelAvatar] = record.video.channelAvatarUrl + this[HistoryTable.duration] = record.video.durationSeconds + this[HistoryTable.progress] = record.positionSeconds + this[HistoryTable.watchedAt] = record.watchedAt + } + return records.size.toLong() + } + + private data class HistoryKey(val url: String, val watchedAt: Long) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt index 1fa94f38..9f365481 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt @@ -6,19 +6,20 @@ internal object TypeTypePortabilityImport { category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long = when (category) { PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.SUBSCRIPTION_GROUPS, - PortabilityCategory.HISTORY, PortabilityCategory.PLAYLISTS, - -> TypeTypePortabilityCoreImport.write(userId, category, source, policy) + -> TypeTypePortabilityCoreImport.write(userId, category, source, policy, onRecord) + PortabilityCategory.HISTORY -> TypeTypePortabilityHistoryImport.write(userId, source, policy, onRecord) PortabilityCategory.WATCH_LATER, PortabilityCategory.FAVORITES, PortabilityCategory.PROGRESS, PortabilityCategory.SEARCH_HISTORY, PortabilityCategory.SAVED_PLAYLISTS, - -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy) - PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source) - PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy) + -> TypeTypePortabilityLibraryImport.write(userId, category, source, policy, onRecord) + PortabilityCategory.SETTINGS -> TypeTypePortabilitySettingsImport.write(userId, source, onRecord) + PortabilityCategory.CONTENT_FILTERS -> TypeTypePortabilityFilterImport.write(userId, source, policy, onRecord) } } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt index d1b3ae65..cd9a4ed0 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt @@ -19,6 +19,7 @@ internal object TypeTypePortabilityLibraryImport { category: PortabilityCategory, source: PortabilityRecordSource, policy: PortabilityDuplicatePolicy, + onRecord: () -> Unit, ): Long { clearIfReplacing(userId, category, policy) var count = 0L @@ -31,6 +32,7 @@ internal object TypeTypePortabilityLibraryImport { is PortabilitySavedPlaylist -> savedPlaylist(userId, record) else -> 0 } + onRecord() } return count } diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt index 9479bf65..5f457278 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt @@ -5,12 +5,14 @@ import dev.typetype.server.models.SettingsItem import dev.typetype.server.services.TypeTypeBackupLibraryRestore internal object TypeTypePortabilitySettingsImport { - fun write(userId: String, source: PortabilityRecordSource): Long { + fun write(userId: String, source: PortabilityRecordSource, onRecord: () -> Unit): Long { var count = 0L source.forEach(PortabilityCategory.SETTINGS) { record -> - if (record !is PortabilitySettings) return@forEach - val settings = CacheJson.decodeFromString(record.values.toString()) - count += TypeTypeBackupLibraryRestore.settings(userId, settings) + if (record is PortabilitySettings) { + val settings = CacheJson.decodeFromString(record.values.toString()) + count += TypeTypeBackupLibraryRestore.settings(userId, settings) + } + onRecord() } return count } diff --git a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt index 56ff39a1..45b69bad 100644 --- a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt +++ b/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt @@ -37,6 +37,7 @@ class PortabilityArchiveReproductionTest { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map = error("Account writes are forbidden in this probe") override suspend fun export( diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt index babd1b5a..11ce8800 100644 --- a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt @@ -152,8 +152,13 @@ private object RouteDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ) = source.counts().mapKeys { it.key.wireName }.also { result -> - request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + request.categories.forEach { category -> + val count = result[category.wireName] ?: 0L + onCategoryProgress(category, count) + onCategoryComplete(category, count) + } } override suspend fun export( diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt index 4df0f853..83298256 100644 --- a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt @@ -41,6 +41,9 @@ class PortabilityEngineTest { ) val completed = awaitState(engine, "owner-a", started.id, PortabilityJobState.COMPLETED) assertEquals(1L, completed.result?.get("subscriptions")) + assertEquals(PortabilityProgressUnit.RECORDS, completed.progress?.unit) + assertEquals(1L, completed.progress?.processed) + assertEquals(1L, completed.progress?.total) engine.close() } @@ -187,8 +190,13 @@ private class FakeDataPort : PortabilityDataPort { source: PortabilityRecordSource, request: PortabilityImportRequest, onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, ): Map = source.counts().mapKeys { it.key.wireName }.also { result -> - request.categories.forEach { category -> onCategoryComplete(category, result[category.wireName] ?: 0L) } + request.categories.forEach { category -> + val count = result[category.wireName] ?: 0L + onCategoryProgress(category, count) + onCategoryComplete(category, count) + } } override suspend fun export( diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt new file mode 100644 index 00000000..74a9da40 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt @@ -0,0 +1,85 @@ +package dev.typetype.server.portability + +import dev.typetype.server.TEST_USER_ID +import dev.typetype.server.TestDatabase +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.HistoryTable +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.UUID + +class TypeTypePortabilityHistoryImportTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `history import batches records and skips existing and source duplicates`() = runBlocking { + insertExisting(history(0)) + val records = (0..1_000).map(::history) + history(1_000) + var processed = 0L + + val imported = DatabaseFactory.query { + TypeTypePortabilityHistoryImport.write( + TEST_USER_ID, + Source(records), + PortabilityDuplicatePolicy.SKIP, + ) { processed++ } + } + + assertEquals(1_000L, imported) + assertEquals(1_002L, processed) + assertEquals(1_001L, transaction { HistoryTable.selectAll().count() }) + } + + private suspend fun insertExisting(record: PortabilityHistory) { + DatabaseFactory.query { + HistoryTable.insert { + it[HistoryTable.id] = UUID.randomUUID().toString() + it[HistoryTable.userId] = TEST_USER_ID + it[HistoryTable.url] = record.video.url + it[HistoryTable.title] = record.video.title + it[HistoryTable.thumbnail] = record.video.thumbnailUrl + it[HistoryTable.channelName] = record.video.channelName + it[HistoryTable.channelUrl] = record.video.channelUrl + it[HistoryTable.channelAvatar] = record.video.channelAvatarUrl + it[HistoryTable.duration] = record.video.durationSeconds + it[HistoryTable.progress] = record.positionSeconds + it[HistoryTable.watchedAt] = record.watchedAt + } + } + } + + private fun history(index: Int) = PortabilityHistory( + video = PortabilityVideo( + url = "https://video.example/$index", + title = "Video $index", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://channel.example", + ), + watchedAt = index.toLong(), + ) +} + +private class Source(private val records: List) : PortabilityRecordSource { + override fun categories() = setOf(PortabilityCategory.HISTORY) + + override fun counts() = mapOf(PortabilityCategory.HISTORY to records.size.toLong()) + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + if (category == PortabilityCategory.HISTORY) records.forEach(block) + } +} From 8751877542ee9d085ddc4702c4cce8085f1c7f96 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 00:43:13 +0200 Subject: [PATCH 011/137] fix: support localized YouTube Takeout schemas --- .../YoutubeTakeoutCsvPortabilityReader.kt | 38 +++++++-- .../YoutubeTakeoutActivityClassifier.kt | 85 ++++++++++++++++++- .../services/YoutubeTakeoutSchemaHints.kt | 80 ++++++++--------- .../services/YoutubeTakeoutSchemaTerms.kt | 43 ++++++++++ .../services/YoutubeTakeoutSystemPlaylist.kt | 21 +++++ .../services/YoutubeTakeoutZipScanner.kt | 16 ++-- 6 files changed, 226 insertions(+), 57 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index 14577074..fe07606e 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -14,7 +14,7 @@ internal object YoutubeTakeoutCsvPortabilityReader { private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" fun readManifests(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - csvEntries(entries).filter(::isPlaylistManifest).forEach { entry -> + csvEntries(entries).filter { isPlaylistManifest(zip, it) }.forEach { entry -> sink.markCategory(PortabilityCategory.PLAYLISTS) forEachRow(zip, entry) { header, row -> val playlist = YoutubeTakeoutRowParser.parsePlaylist(header, row) ?: return@forEachRow @@ -27,7 +27,7 @@ internal object YoutubeTakeoutCsvPortabilityReader { } fun readContent(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - val csv = csvEntries(entries).filterNot(::isPlaylistManifest).toList() + val csv = csvEntries(entries).filterNot { isPlaylistManifest(zip, it) }.toList() csv.filter(::isSubscriptionFile).forEach { readSubscriptions(zip, it, sink) } csv.filter(::isPlaylistContent).forEach { readPlaylistItems(zip, it, sink) } } @@ -99,13 +99,25 @@ internal object YoutubeTakeoutCsvPortabilityReader { it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() } - private fun isPlaylistManifest(entry: ZipEntry): Boolean = fileStem(entry) in setOf("playlists", "oynatma listeleri") + private fun isPlaylistManifest(zip: ZipFile, entry: ZipEntry): Boolean { + if (YoutubeTakeoutSchemaHints.isPlaylistManifestName(fileStem(entry))) return true + if (!YoutubeTakeoutSchemaHints.isPlaylistText(entry.name)) return false + var manifest = false + forEachRow(zip, entry) { header, _ -> + manifest = manifest || ( + header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) && + header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) && + header.none(YoutubeTakeoutSchemaHints::isVideoIdHeader) + ) + } + return manifest + } - private fun isSubscriptionFile(entry: ZipEntry): Boolean = fileStem(entry) in SUBSCRIPTION_NAMES + private fun isSubscriptionFile(entry: ZipEntry): Boolean = + YoutubeTakeoutSchemaHints.isSubscriptionText(fileStem(entry)) private fun isPlaylistContent(entry: ZipEntry): Boolean { - val normalized = YoutubeTakeoutSchemaHints.normalize(entry.name) - return "playlist" in normalized || "oynatma list" in normalized + return YoutubeTakeoutSchemaHints.isPlaylistText(entry.name) } private fun playlistKeyFromPath(path: String): String? = path.substringAfterLast('/').substringBeforeLast('.') @@ -123,13 +135,25 @@ internal object YoutubeTakeoutCsvPortabilityReader { name, "Videos from $name", "Videos de $name", + "Vídeos de $name", "Vidéos de $name", "Videos da playlist $name", + "Vídeos da playlist $name", + "Video di $name", + "Video da $name", + "Videos von $name", + "Videos van $name", + "Filmy z playlisty $name", + "Видео из $name", + "動画 - $name", + "$name の動画", + "$name 동영상", + "$name 视频", + "$name 影片", "$name videos", ) private fun PortabilityRecordSink.invalid(category: PortabilityCategory, kind: String) = issue(PortabilityIssue(category, "invalid_takeout_row", "An invalid YouTube Takeout $kind row was skipped")) - private val SUBSCRIPTION_NAMES = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt index 6a01dcfe..d7c91820 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt @@ -5,15 +5,48 @@ object YoutubeTakeoutActivityClassifier { "You watched", "Vous avez regardé", "Has visto", + "Has visto el vídeo", + "Ha visto", "Você assistiu", + "Você assistiu a", + "Você assistiu ao vídeo", "hai guardato", + "Hai guardato", + "Hai guardato il video", "İzlediniz", "du hast angesehen", + "Du hast dir angesehen", + "Du hast dir das angesehen", + "je hebt gekeken", + "je hebt gekeken naar", + "obejrzałeś", + "obejrzałaś", + "obejrzano", "вы смотрели", + "вы посмотрели", + "ви дивилися", + "ви переглянули", + "du tittade på", + "du har sett", + "du så på", + "katsoit", + "sledovali jste", + "ai vizionat", + "anda menonton", + "bạn đã xem", + "आपने देखा", + "あなたが視聴した動画", + "視聴した動画", "視聴しました", + "動画を視聴しました", "시청한 동영상", + "동영상을 시청했습니다", "已观看", "已觀看", + "观看了", + "觀看了", + "شاهدت", + "لقد شاهدت", ) private val likedPhrases = setOf( @@ -22,15 +55,36 @@ object YoutubeTakeoutActivityClassifier { "Vous avez aimé", "A aimé", "te ha gustado", + "te gustó", + "ha gustado", "Você gostou", + "Você curtiu", + "Você marcou como gostei", "hai messo mi piace", + "ti è piaciuto", "Beğendiniz", "gefällt mir", + "dir gefällt", + "je hebt dit leuk gevonden", + "podobał ci się", "понравилось", + "вы поставили отметку нравится", + "сподобалося", + "du gillade", + "du kunne godt lide", + "du likte", + "pidit", + "líbilo se vám", + "ți-a plăcut", + "anda menyukai", + "bạn đã thích", + "आपको पसंद आया", "高く評価しました", "좋아요 표시함", "点赞了", "按讚", + "您喜欢了", + "أعجبك", ) private val subscribedPhrases = setOf( @@ -38,21 +92,46 @@ object YoutubeTakeoutActivityClassifier { "Vous vous êtes abonné à", "te has suscrito a", "Você se inscreveu em", + "Você se inscreveu no canal", "ti sei iscritto a", + "ti sei iscritta a", "Abone oldunuz", "du hast abonniert", + "du hast den kanal abonniert", + "je hebt je geabonneerd op", + "zasubskrybowałeś", + "zasubskrybowałaś", "вы подписались на", + "вы подписались на канал", + "ви підписалися на", + "du prenumererade på", + "du abonnerede på", + "du abonnerte på", + "tilasit", + "te-ai abonat la", + "anda berlangganan", + "bạn đã đăng ký", + "आपने सदस्यता ली", "登録しました", + "チャンネル登録しました", "구독함", + "구독했습니다", "已订阅", "已訂閱", + "订阅了", + "訂閱了", + "اشتركت في", + "لقد اشتركت في", ) + private val normalizedWatchedPhrases = watchedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedLikedPhrases = likedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedSubscribedPhrases = subscribedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) - fun isWatched(value: String): Boolean = containsAny(value, watchedPhrases) + fun isWatched(value: String): Boolean = containsAny(value, normalizedWatchedPhrases) - fun isLiked(value: String): Boolean = containsAny(value, likedPhrases) + fun isLiked(value: String): Boolean = containsAny(value, normalizedLikedPhrases) - fun isSubscribed(value: String): Boolean = containsAny(value, subscribedPhrases) + fun isSubscribed(value: String): Boolean = containsAny(value, normalizedSubscribedPhrases) val watchedPattern: String = pattern(watchedPhrases) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt index e2a784c7..8d4e67b2 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt @@ -5,38 +5,16 @@ object YoutubeTakeoutSchemaHints { private val videoIdRegex = Regex("""^[A-Za-z0-9_-]{6,}$""") private val playlistIdRegex = Regex("""^(PL|UU|LL|RD|OLAK5uy_)[A-Za-z0-9_-]{6,}$""") private val channelUrlRegex = Regex("""youtube\.com/(channel/|@|c/|user/)""", RegexOption.IGNORE_CASE) - private val titleWords = setOf( - "title", - "titles", - "titre", - "titres", - "titulo", - "titulos", - "baslik", - "baslık", - "başlık", - "basligi", - "baslıgı", - "başlığı", - "adi", - "adı", - "nombre", - "nome", - "nom", - "name", - "names", - "titel", - "titolo", - ) - private val playlistWords = setOf("playlist", "playlists", "oynatma listesi", "oynatma listeleri") - private val channelWords = setOf("channel", "chaine", "canal", "kanal", "kanaal", "канал", "チャンネル", "채널", "频道", "頻道") - + private val titleWords = YoutubeTakeoutSchemaTerms.titleWords + private val playlistWords = YoutubeTakeoutSchemaTerms.playlistWords + private val channelWords = YoutubeTakeoutSchemaTerms.channelWords + private val videoWords = YoutubeTakeoutSchemaTerms.videoWords fun isChannelIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "channel id" || normalized == "id des chaines" || normalized == "id de la chaine" || - (channelWords.any { it in normalized } && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (channelWords.any { normalize(it) in normalized } && hasIdentifier(normalized)) } fun isChannelUrlHeader(value: String): Boolean { @@ -44,47 +22,48 @@ object YoutubeTakeoutSchemaHints { return normalized == "channel url" || normalized == "url des chaines" || normalized == "url de la chaine" || - ("url" in words(normalized) && channelWords.any { it in normalized }) + ("url" in words(normalized) && channelWords.any { normalize(it) in normalized }) } fun isChannelTitleHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "titres des chaines" || normalized == "titre de la chaine" || - titleWords.any { it in words(normalized) } && - (channelWords.any { it in normalized } || normalized in titleWords) + titleWords.any { normalize(it) in normalized } && + (channelWords.any { normalize(it) in normalized } || titleWords.any { normalize(it) == normalized }) } fun isPlaylistIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "playlist id" || normalized == "id de la playlist" || - (isPlaylistText(normalized) && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (isPlaylistText(normalized) && hasIdentifier(normalized)) } fun isPlaylistTitleHeader(value: String): Boolean { val normalized = normalize(value) - return isPlaylistText(normalized) && titleWords.any { it in words(normalized) } + return isPlaylistText(normalized) && titleWords.any { normalize(it) in normalized } } fun isVideoIdHeader(value: String): Boolean { val normalized = normalize(value) return normalized == "video id" || normalized == "id video" || - ("video" in normalized && ("id" in words(normalized) || "kimligi" in words(normalized) || "kimliği" in words(normalized))) + (videoWords.any { normalize(it) in normalized } && hasIdentifier(normalized)) } fun isVideoTitleHeader(value: String): Boolean { val normalized = normalize(value) - return "video" in normalized && titleWords.any { it in words(normalized) } + return videoWords.any { normalize(it) in normalized } && titleWords.any { normalize(it) in normalized } } fun isPlaylistItemAddedAtHeader(value: String): Boolean { val normalized = normalize(value) - val parts = words(normalized) - return ("added" in parts && parts.any { it == "at" || it == "date" || it == "time" || it == "timestamp" }) || - ("creation" in parts && "video" in parts) || - ("created" in parts && "at" in parts) || - ("timestamp" in parts && ("playlist" in normalized || "video" in normalized)) + val hasTemporalWord = TEMPORAL_WORDS.any { normalize(it) in normalized } + return hasTemporalWord && ( + isPlaylistText(normalized) || videoWords.any { normalize(it) in normalized } || + "added at" in normalized || "date added" in normalized || "created at" in normalized || + normalized == "timestamp" + ) } fun isUrlHeader(value: String): Boolean = "url" in words(normalize(value)) @@ -112,5 +91,28 @@ object YoutubeTakeoutSchemaHints { return playlistWords.any { normalize(it) in normalized } } + fun isPlaylistManifestName(value: String): Boolean = + YoutubeTakeoutSchemaTerms.playlistManifestNames.contains(normalize(value)) + + fun isSubscriptionText(value: String): Boolean = + YoutubeTakeoutSchemaTerms.subscriptionWords.any { normalize(it) in normalize(value) } + + private fun hasIdentifier(normalized: String): Boolean { + val parts = words(normalized) + return "id" in parts || "identifier" in parts || "identificador" in parts || + "identifiant" in parts || "kimligi" in parts || "kimliği" in parts || + "идентификатор" in normalized || "identyfikator" in parts || "identifikator" in parts || + "kennung" in parts || "識別子" in normalized || "標識符" in normalized || + "标识符" in normalized || "معرف" in normalized + } + private fun words(value: String): Set = value.split(' ').filter { it.isNotBlank() }.toSet() + + private val TEMPORAL_WORDS = setOf( + "added", "date", "time", "timestamp", "creation", "created", "fecha", "marca de tiempo", "creacion", + "ajoute", "datum", "zeit", "erstellt", "data", "ora", "creazione", "criacao", "eklenme", "olusturma", + "дата", "время", "создания", "добавления", "日時", "时间", "日期", "创建", "추가", "생성", "التاريخ", "وقت", + "إضافة", "बनाने", "समय", "ημερομηνία", "เวลา", + ) + } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt new file mode 100644 index 00000000..bbeea247 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.services + +internal object YoutubeTakeoutSchemaTerms { + val titleWords = setOf( + "title", "titles", "titre", "titres", "titulo", "titulos", "baslik", "baslık", "başlık", + "basligi", "baslıgı", "başlığı", "adi", "adı", "nombre", "nome", "nom", "name", "names", + "titel", "titolo", "titulo original", "tytul", "название", "наименование", "название видео", + "動画タイトル", "動画のタイトル", "제목", "视频标题", "影片標題", "naam", "عنوان", "शीर्षक", "τίτλος", + ) + + val playlistWords = setOf( + "playlist", "playlists", "oynatma listesi", "oynatma listeleri", "playlistler", + "lista de reproduccion", "listas de reproduccion", "liste de lecture", "listes de lecture", + "lista di riproduzione", "liste di riproduzione", "lista de reproducao", "listas de reproducao", + "wiedergabeliste", "wiedergabelisten", "afspeellijst", "afspeellijsten", + "список воспроизведения", "списки воспроизведения", "плейлист", "плейлисты", "再生リスト", + "재생목록", "播放列表", "播放清單", "قائمة التشغيل", "قوائم التشغيل", "प्लेलिस्ट", "प्लेलिस्टें", + "λίστες αναπαραγωγής", + ) + + val channelWords = setOf( + "channel", "chaine", "canal", "canale", "kanal", "kanaal", "канал", "канали", "kanał", + "チャンネル", "채널", "频道", "頻道", "قناة", "चैनल", "κανάλι", + ) + + val videoWords = setOf( + "video", "videos", "vidéo", "vidéos", "vídeo", "vídeos", "動画", "동영상", "비디오", "видео", + "视频", "影片", "film", "filmy", "فيديو", "فيديوهات", "वीडियो", "βίντεο", "วิดีโอ", + ) + + val subscriptionWords = setOf( + "subscriptions", "suscripciones", "inscricoes", "inscrições", "iscrizioni", "abbonamenti", "abos", + "abonelikler", "abonnements", "abonnementen", "abonnierte kanäle", "subskrypcje", "подписки", + "підписки", "登録チャンネル", "チャンネル登録", "구독", "订阅", "訂閱", "الاشتراكات", + ) + + val playlistManifestNames = setOf( + "playlists", "oynatma listeleri", "oynatma listesi", "listas de reproduccion", "listes de lecture", + "liste di riproduzione", "listas de reproducao", "wiedergabelisten", "wiedergabeliste", + "afspeellijsten", "afspeellijst", "списки воспроизведения", "список воспроизведения", "再生リスト", + "재생목록", "播放列表", "播放清單", "قوائم التشغيل", "قائمة التشغيل", + ).mapTo(mutableSetOf(), YoutubeTakeoutTextNormalizer::normalize) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt index 791bc126..13dcaa89 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt @@ -13,11 +13,21 @@ object YoutubeTakeoutSystemPlaylist { "daha sonra izlenecekler", "spater ansehen", "später ansehen", + "later bekijken", + "obejrzyj pozniej", "посмотреть позже", + "смотреть позже", "後で見る", "나중에 볼 동영상", "稍后观看", "稍後觀看", + "بعدا تماشا کنید", + "مشاهدة لاحقا", + "المشاهدة لاحقا", + "переглянути пізніше", + "बाद में देखें", + "ดูภายหลัง", + "παρακολούθηση αργότερα", ) private val likedPhrases = setOf( @@ -28,7 +38,10 @@ object YoutubeTakeoutSystemPlaylist { "videos que me gustan", "videos que gostei", "videos curtidos", + "videos de que gostei", + "videos que eu gostei", "video piaciuti", + "video che mi piacciono", "begendigim videolar", "begendiğim videolar", "beğendigim videolar", @@ -37,11 +50,19 @@ object YoutubeTakeoutSystemPlaylist { "beğenilen videolar", "mit gefallt", "gefällt mir", + "videos die mir gefallen", + "leuke videos", + "polubione filmy", "понравившиеся видео", + "понравились видео", "高く評価した動画", "좋아요 표시한 동영상", "喜欢的视频", "喜歡的影片", + "الفيديوهات التي أعجبتني", + "वीडियो जो मुझे पसंद हैं", + "βίντεο που μου αρέσουν", + "улюблені відео", ) fun canonicalKey(value: String): String? { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt index c1fe13ec..d940111f 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt @@ -69,12 +69,15 @@ object YoutubeTakeoutZipScanner { private fun isPlaylistsHeader(path: String, header: List, rows: List>): Boolean { val hasId = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikePlaylistId) val hasTitle = header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) - return hasId && hasTitle && isMainPlaylistsFile(path) + return hasId && hasTitle && (isMainPlaylistsFile(path) || isPlaylistPath(path)) } private fun isPlaylistItemsEntry(path: String, header: List, rows: List>): Boolean { - if ((!isPlaylistPath(path) && !isPlaylistItemsFile(path)) || isMainPlaylistsFile(path)) return false - return header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeVideoId) + if (isMainPlaylistsFile(path)) return false + val hasVideo = header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeVideoId) + val hasPlaylistKey = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + return hasVideo && (isPlaylistPath(path) || isPlaylistItemsFile(path) || hasPlaylistKey) } private fun extractPlaylistSourceKey(path: String, header: List): String? { @@ -84,8 +87,7 @@ object YoutubeTakeoutZipScanner { } private fun isSubscriptionPath(path: String): Boolean { - val normalized = YoutubeTakeoutSchemaHints.normalize(path) - return SUBSCRIPTION_PATH_MARKERS.any { it in normalized } + return YoutubeTakeoutSchemaHints.isSubscriptionText(path) } private fun isPlaylistPath(path: String): Boolean = YoutubeTakeoutSchemaHints.isPlaylistText(path) @@ -95,11 +97,9 @@ object YoutubeTakeoutZipScanner { private fun isMainPlaylistsFile(path: String): Boolean = path.substringAfterLast('/').substringBeforeLast('.').let { fileName -> - val normalized = YoutubeTakeoutSchemaHints.normalize(fileName) - normalized == "playlists" || normalized == "oynatma listeleri" + YoutubeTakeoutSchemaHints.isPlaylistManifestName(fileName) } private fun List>.hasValue(predicate: (String) -> Boolean): Boolean = any { row -> row.any(predicate) } - private val SUBSCRIPTION_PATH_MARKERS = setOf("subscriptions", "abonnements", "suscripciones", "inscricoes", "abos", "abonelikler") } From 8fe4962264f8e64a47b0f15adf9e14b0bd4c3427 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 00:43:24 +0200 Subject: [PATCH 012/137] perf: accelerate localized Takeout parsing --- .../YoutubeTakeoutHtmlPortabilityReader.kt | 36 +++-- .../YoutubeTakeoutActivitySignalService.kt | 61 +++++---- .../services/YoutubeTakeoutDateParser.kt | 123 ++++++++++++++++-- .../services/YoutubeTakeoutHistoryParser.kt | 33 +++-- .../services/YoutubeTakeoutParserService.kt | 12 +- .../services/YoutubeTakeoutPathHints.kt | 33 +++++ .../services/YoutubeTakeoutTextNormalizer.kt | 10 +- 7 files changed, 243 insertions(+), 65 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt index e48116cf..73636bbf 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -2,6 +2,7 @@ package dev.typetype.server.portability import dev.typetype.server.services.YoutubeTakeoutActivitySignalService import dev.typetype.server.services.YoutubeTakeoutHistoryParser +import dev.typetype.server.services.YoutubeTakeoutPathHints import java.io.Reader import java.util.zip.ZipEntry import java.util.zip.ZipFile @@ -10,16 +11,29 @@ internal object YoutubeTakeoutHtmlPortabilityReader { fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> zip.getInputStream(entry).bufferedReader().use { reader -> - readWindows(reader) { html -> writeWindow(html, sink) } + val historyEntry = isHistoryEntry(entry) + val write = { html: String -> + writeWindow(html, sink, includeActivitySignals = !historyEntry, historyEntry) + } + if (entry.size in 1..WHOLE_ENTRY_BYTES) write(reader.readText()) + else readWindows(reader, write) } } } - private fun writeWindow(html: String, sink: PortabilityRecordSink) { - YoutubeTakeoutHistoryParser.parse(html).forEach { sink.write(it.toPortability()) } - val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) - subscriptions.forEach { sink.write(it.toPortability()) } - favorites.forEach { sink.write(it.toPortability()) } + private fun writeWindow( + html: String, + sink: PortabilityRecordSink, + includeActivitySignals: Boolean, + historyEntry: Boolean, + ) { + YoutubeTakeoutHistoryParser.parse(html, requireWatchedMarker = !historyEntry) + .forEach { sink.write(it.toPortability()) } + if (includeActivitySignals) { + val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) + subscriptions.forEach { sink.write(it.toPortability()) } + favorites.forEach { sink.write(it.toPortability()) } + } } private fun readWindows(reader: Reader, block: (String) -> Unit) { @@ -37,10 +51,12 @@ internal object YoutubeTakeoutHtmlPortabilityReader { if (window.isNotEmpty()) block(window.toString()) } - private fun isYoutubeHtml(entry: ZipEntry): Boolean = - entry.name.endsWith(".html", ignoreCase = true) && "youtube" in entry.name.lowercase() + private fun isYoutubeHtml(entry: ZipEntry): Boolean = YoutubeTakeoutPathHints.isYoutubeHtml(entry.name) + + private fun isHistoryEntry(entry: ZipEntry): Boolean = YoutubeTakeoutPathHints.isHistoryEntry(entry.name) private const val READ_CHARS = 32 * 1024 - private const val WINDOW_CHARS = 512 * 1024 - private const val OVERLAP_CHARS = 128 * 1024 + private const val WINDOW_CHARS = 4 * 1024 * 1024 + private const val OVERLAP_CHARS = 256 * 1024 + private const val WHOLE_ENTRY_BYTES = 128L * 1024L * 1024L } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index 4846bce3..cbbabeaf 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -6,8 +6,8 @@ import java.nio.file.Path import java.util.zip.ZipFile object YoutubeTakeoutActivitySignalService { - private val subscribedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.subscribedPattern})\s*([\s\S]*?)
""", RegexOption.IGNORE_CASE) - private val likedRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.likedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) + private val activityLinkRegex = Regex("""]*>([^<]*)""", RegexOption.IGNORE_CASE) + private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val channelUrlRegex = Regex("""https?://www\.youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+)""") private val spacesRegex = Regex("""\s+""") @@ -17,8 +17,7 @@ object YoutubeTakeoutActivitySignalService { val favorites = mutableListOf() ZipFile(zipPath.toFile()).use { zip -> zip.entries().asSequence().forEach { entry -> - val normalized = entry.name.lowercase() - if (entry.isDirectory || !normalized.endsWith(".html") || !normalized.contains("youtube")) return@forEach + if (entry.isDirectory || !YoutubeTakeoutPathHints.isYoutubeHtml(entry.name) || isHistoryEntry(entry.name)) return@forEach val html = zip.getInputStream(entry).bufferedReader().use { it.readText() }.replace("\u00a0", " ") val parsed = parseHtml(html) subscriptions += parsed.first @@ -29,31 +28,37 @@ object YoutubeTakeoutActivitySignalService { } internal fun parseHtml(html: String): Pair, List> = - parseSubscriptions(html) to parseFavorites(html) + parseLinks(html) - private fun parseSubscriptions(html: String): List { - return subscribedRegex.findAll(html).mapNotNull { match -> - val url = channelUrlRegex.find(decode(match.groupValues[1]))?.value ?: return@mapNotNull null - val name = decode(match.groupValues[2]) - SubscriptionItem(channelUrl = url.replace("http://", "https://"), name = name, avatarUrl = "") - }.toList() - } - - private fun parseFavorites(html: String): List { - return likedRegex.findAll(html).mapNotNull { match -> - val title = decode(match.groupValues[2]) - if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null - val source = decode(match.groupValues[1]) + " " + title - val videoUrl = watchUrlRegex.find(source)?.value?.replace("http://", "https://") - ?: return@mapNotNull null - FavoriteItem( + private fun parseLinks(html: String): Pair, List> { + val resolvedHtml = html.replace("\u00a0", " ") + val subscriptions = mutableListOf() + val favorites = mutableListOf() + activityLinkRegex.findAll(resolvedHtml).forEach { match -> + val prefixStart = (match.range.first - PREFIX_CHARS).coerceAtLeast(0) + val prefix = resolvedHtml.substring(prefixStart, match.range.first) + val href = decode(match.groupValues[1]) + val text = decode(match.groupValues[2]) + channelUrlRegex.find(href)?.value?.let { channelUrl -> + if (YoutubeTakeoutActivityClassifier.isSubscribed(prefix)) { + subscriptions += SubscriptionItem(channelUrl.replace("http://", "https://"), text, "") + } + } + val videoUrl = watchUrlRegex.find(href)?.value?.replace("http://", "https://") ?: return@forEach + if (!YoutubeTakeoutActivityClassifier.isLiked(prefix) || YoutubeTakeoutUnavailableItem.matches(text)) return@forEach + val tailStart = match.range.last + 1 + val tail = rowTailRegex.find(resolvedHtml, tailStart)?.takeIf { it.range.first == tailStart } ?: return@forEach + val favoritedAt = YoutubeTakeoutDateParser.parseEpochMillis(decode(tail.groupValues[3])) + ?: return@forEach + favorites += FavoriteItem( videoUrl = videoUrl, - favoritedAt = YoutubeTakeoutDateParser.parseEpochMillis(decode(match.groupValues[5])) ?: 0L, - title = title, - channelName = decode(match.groupValues[4]), - channelUrl = decode(match.groupValues[3]).replace("http://", "https://"), + favoritedAt = favoritedAt, + title = text, + channelName = decode(tail.groupValues[2]), + channelUrl = decode(tail.groupValues[1]).replace("http://", "https://"), ) - }.toList() + } + return subscriptions to favorites } private fun decode(value: String): String { @@ -67,4 +72,8 @@ object YoutubeTakeoutActivitySignalService { .replace(spacesRegex, " ") .trim() } + + private fun isHistoryEntry(name: String): Boolean = YoutubeTakeoutPathHints.isHistoryEntry(name) + + private const val PREFIX_CHARS = 160 } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt index 2364cdc1..44ebd295 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -2,21 +2,47 @@ package dev.typetype.server.services import java.time.Instant import java.time.LocalDate +import java.time.Month import java.time.OffsetDateTime import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter +import java.time.format.DateTimeFormatterBuilder +import java.time.format.TextStyle import java.util.Locale object YoutubeTakeoutDateParser { - private val activityFormatters = listOf( - Locale.ENGLISH, - Locale.FRENCH, - ).flatMap { locale -> - listOf("d MMM yyyy, HH:mm:ss z", "d MMMM yyyy, HH:mm:ss z").map { pattern -> - DateTimeFormatter.ofPattern(pattern, locale) - } + private val ACTIVITY_PATTERNS = listOf( + "d MMM yyyy, HH:mm:ss z", + "d MMMM yyyy, HH:mm:ss z", + "d. MMM yyyy, HH:mm:ss z", + "d. MMMM yyyy, HH:mm:ss z", + "MMM d, yyyy, HH:mm:ss z", + "MMMM d, yyyy, HH:mm:ss z", + "d MMM yyyy, h:mm:ss a z", + "d MMMM yyyy, h:mm:ss a z", + "MMM d, yyyy, h:mm:ss a z", + "MMMM d, yyyy, h:mm:ss a z", + "d MMM yyyy HH:mm:ss z", + "d MMMM yyyy HH:mm:ss z", + "MMM d, yyyy HH:mm:ss z", + "MMMM d, yyyy HH:mm:ss z", + "d/M/uuuu, HH:mm:ss z", + "M/d/uuuu, HH:mm:ss z", + "yyyy年M月d日 HH:mm:ss z", + "yyyy年M月d日 H:mm:ss z", + "yyyy년 M월 d일 HH:mm:ss z", + "yyyy년 M월 d일 H:mm:ss z", + "d M月 yyyy, HH:mm:ss z", + "d M月 yyyy HH:mm:ss z", + "d M월 yyyy, HH:mm:ss z", + "d M월 yyyy HH:mm:ss z", + ) + private val canonicalActivityFormatters = ACTIVITY_PATTERNS.map(::formatter) + private val activityFormatters by lazy { + localeCandidates().flatMap { locale -> ACTIVITY_PATTERNS.map { formatter(it, locale) } } } + private val monthAliases by lazy(::buildMonthAliases) fun parseEpochMillis(value: String): Long? { val trimmed = value.replace("\u00a0", " ").trim() @@ -40,11 +66,84 @@ object YoutubeTakeoutDateParser { }.getOrNull() private fun parseActivityDate(value: String): Long? { - activityFormatters.forEach { formatter -> - runCatching { ZonedDateTime.parse(value, formatter).toInstant().toEpochMilli() } - .getOrNull() - ?.let { return it } + val normalized = normalizeActivityText(value) + val canonical = canonicalizeMonths(normalized) + canonicalActivityFormatters.asSequence() + .mapNotNull { parseWith(canonical, it) } + .firstOrNull() + ?.let { return it } + if (!hasKnownMonth(normalized)) return null + return activityFormatters.asSequence().mapNotNull { parseWith(normalized, it) }.firstOrNull() + } + + private fun hasKnownMonth(value: String): Boolean = MONTH_TOKEN_REGEX.findAll(value) + .any { monthAliases.containsKey(normalizeMonth(it.value)) } + + private fun normalizeActivityText(value: String): String = value + .replace('\u060C', ',') + .replace('\uFF0C', ',') + .replace(ACTIVITY_CONNECTOR_REGEX, " ") + .replace(ACTIVITY_SPACES_REGEX, " ") + .trim() + + private fun parseWith(value: String, formatter: DateTimeFormatter): Long? = runCatching { + ZonedDateTime.parse(value, formatter).toInstant().toEpochMilli() + }.getOrNull() + + private fun canonicalizeMonths(value: String): String = MONTH_TOKEN_REGEX.replace(value) { match -> + monthAliases[normalizeMonth(match.value)] ?: match.value + } + + private fun buildMonthAliases(): Map { + val aliases = mutableMapOf>() + localeCandidates().forEach { locale -> + Month.entries.forEach { month -> + TextStyle.entries.filter { it != TextStyle.NARROW }.forEach { style -> + val alias = normalizeMonth(month.getDisplayName(style, locale)) + if (alias.length >= 2) aliases.getOrPut(alias) { mutableSetOf() }.add(month.value) + } + } } - return null + return aliases.mapNotNull { (alias, months) -> + months.singleOrNull()?.let { alias to Month.of(it).getDisplayName(TextStyle.SHORT, Locale.ENGLISH) } + }.toMap() + } + + private fun normalizeMonth(value: String): String = + YoutubeTakeoutTextNormalizer.normalize(value).replace(" ", "") + + private fun localeCandidates(): List { + val preferred = listOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.forLanguageTag("es"), + Locale.GERMAN, + Locale.ITALIAN, + Locale.forLanguageTag("pt"), + Locale.forLanguageTag("tr"), + Locale.forLanguageTag("nl"), + Locale.forLanguageTag("pl"), + Locale.forLanguageTag("ru"), + Locale.forLanguageTag("uk"), + Locale.forLanguageTag("ja"), + Locale.KOREAN, + Locale.SIMPLIFIED_CHINESE, + Locale.TRADITIONAL_CHINESE, + ) + val available = Locale.getAvailableLocales().asSequence() + .filter { it.language.isNotBlank() } + .map { Locale.forLanguageTag(it.language) } + .distinctBy(Locale::getLanguage) + .toList() + return (preferred + available).distinctBy(Locale::toLanguageTag) } + + private fun formatter(pattern: String): DateTimeFormatter = formatter(pattern, Locale.ENGLISH) + + private fun formatter(pattern: String, locale: Locale): DateTimeFormatter = + DateTimeFormatterBuilder().parseCaseInsensitive().appendPattern(pattern).toFormatter(locale) + + private val MONTH_TOKEN_REGEX = Regex("[\\p{L}\\p{M}][\\p{L}\\p{M}.]*") + private val ACTIVITY_CONNECTOR_REGEX = Regex("\\s+(?:de|del)\\s+", RegexOption.IGNORE_CASE) + private val ACTIVITY_SPACES_REGEX = Regex("\\s+") } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index 1cee2b5a..a01390f4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -3,20 +3,35 @@ package dev.typetype.server.services import dev.typetype.server.models.HistoryItem object YoutubeTakeoutHistoryParser { - private val rowRegex = Regex("""(?:${YoutubeTakeoutActivityClassifier.watchedPattern})\s*([\s\S]*?)
\s*(?:([\s\S]*?)
\s*)?([\s\S]*?)
""", RegexOption.IGNORE_CASE) + private val videoLinkRegex = Regex( + """]*>([^<]*)""", + RegexOption.IGNORE_CASE, + ) + private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val tagRegex = Regex("<[^>]+>") private val spacesRegex = Regex("\\s+") - fun parse(html: String): List { + fun parse(html: String, requireWatchedMarker: Boolean = true): List { val resolvedHtml = html.replace("\u00a0", " ") - return rowRegex.findAll(resolvedHtml).mapNotNull { match -> + return videoLinkRegex.findAll(resolvedHtml).mapNotNull { match -> + val prefixStart = (match.range.first - WATCHED_PREFIX_CHARS).coerceAtLeast(0) + if (requireWatchedMarker && !YoutubeTakeoutActivityClassifier.isWatched( + resolvedHtml.substring(prefixStart, match.range.first), + ) + ) { + return@mapNotNull null + } + val tailStart = match.range.last + 1 + val tail = rowTailRegex.find(resolvedHtml, tailStart) + ?.takeIf { it.range.first == tailStart } + ?: return@mapNotNull null val url = extractUrl(match.groupValues[1]) ?: return@mapNotNull null val title = decode(match.groupValues[2]) if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null - val channelUrl = match.groupValues[3].takeIf { it.isNotBlank() }.orEmpty() - val channelName = decode(match.groupValues[4]).ifBlank { "Unknown channel" } - val watchedAt = parseDate(decode(match.groupValues[5])) + val channelUrl = tail.groupValues[1].takeIf { it.isNotBlank() }.orEmpty() + val channelName = decode(tail.groupValues[2]).ifBlank { "Unknown channel" } + val watchedAt = parseDate(decode(tail.groupValues[3])) ?: return@mapNotNull null HistoryItem( url = url, title = title, @@ -31,9 +46,7 @@ object YoutubeTakeoutHistoryParser { }.toList().distinctBy { it.url to it.watchedAt } } - private fun parseDate(value: String): Long { - return YoutubeTakeoutDateParser.parseEpochMillis(value) ?: 0L - } + private fun parseDate(value: String): Long? = YoutubeTakeoutDateParser.parseEpochMillis(value) private fun extractUrl(value: String): String? { val decoded = decode(value) @@ -52,4 +65,6 @@ object YoutubeTakeoutHistoryParser { .replace(">", ">") .replace(spacesRegex, " ") .trim() + + private const val WATCHED_PREFIX_CHARS = 160 } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt index 1196917a..6b36ddb6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt @@ -92,15 +92,17 @@ class YoutubeTakeoutParserService { private fun parseHistory(zipPath: Path, warnings: MutableList): List { ZipFile(zipPath.toFile()).use { zip -> val entries = zip.entries().asSequence().filter { item -> - val normalized = item.name.lowercase() - !item.isDirectory && normalized.endsWith(".html") && normalized.contains("youtube") + !item.isDirectory && YoutubeTakeoutPathHints.isYoutubeHtml(item.name) }.toList() - val entry = entries.firstOrNull { it.name.lowercase().contains("watch-history") } - ?: entries.firstOrNull { it.name.lowercase().contains("monactiv") } + val entry = entries.firstOrNull { YoutubeTakeoutPathHints.isHistoryEntry(it.name) } + ?: entries.firstOrNull { YoutubeTakeoutTextNormalizer.normalize(it.name).contains("monactiv") } ?: entries.firstOrNull() if (entry == null) return emptyList() val html = zip.getInputStream(entry).bufferedReader().use { it.readText() } - val parsed = YoutubeTakeoutHistoryParser.parse(html) + val parsed = YoutubeTakeoutHistoryParser.parse( + html, + requireWatchedMarker = !YoutubeTakeoutPathHints.isHistoryEntry(entry.name), + ) if (parsed.isEmpty()) warnings += "No watch history rows detected" return parsed.map(YoutubeTypeTypeMapper::historyItem) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt new file mode 100644 index 00000000..685757e2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt @@ -0,0 +1,33 @@ +package dev.typetype.server.services + +object YoutubeTakeoutPathHints { + fun isYoutubeHtml(path: String): Boolean = + path.endsWith(".html", ignoreCase = true) && "youtube" in path.lowercase() + + fun isHistoryEntry(path: String): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(path) + return HISTORY_MARKERS.any { it in normalized } + } + + private val HISTORY_MARKERS = setOf( + "watch history", + "historique", + "historico", + "historial", + "cronologia", + "verlauf", + "kijkgeschiedenis", + "historia ogladania", + "izleme gecmisi", + "история просмотров", + "история просмотра", + "історія перегляду", + "視聴履歴", + "시청 기록", + "观看记录", + "觀看記錄", + "سجل المشاهدة", + "تاریخچه تماشا", + "देखने का इतिहास", + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt index 5bfe477b..4520edf7 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt @@ -1,11 +1,15 @@ package dev.typetype.server.services import java.text.Normalizer +import java.util.Locale object YoutubeTakeoutTextNormalizer { fun normalize(value: String): String = Normalizer - .normalize(value.lowercase(), Normalizer.Form.NFD) - .replace(Regex("\\p{M}+"), "") - .replace(Regex("[^\\p{L}\\p{N}]+"), " ") + .normalize(value.lowercase(Locale.ROOT), Normalizer.Form.NFD) + .replace(MARKS_REGEX, "") + .replace(NON_ALPHANUMERIC_REGEX, " ") .trim() + + private val MARKS_REGEX = Regex("\\p{M}+") + private val NON_ALPHANUMERIC_REGEX = Regex("[^\\p{L}\\p{N}]+") } From 0113723fb9f54831560b8cd9e570b4c72b94f0a1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 00:43:29 +0200 Subject: [PATCH 013/137] test: cover localized Takeout imports --- ...YoutubeTakeoutActivitySignalServiceTest.kt | 13 ++++ .../server/YoutubeTakeoutParserServiceTest.kt | 71 +++++++++++++++++++ .../YoutubeTakeoutPortabilityAdapterTest.kt | 45 ++++++++++++ 3 files changed, 129 insertions(+) diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt index 845f712d..c5ae9bca 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt @@ -44,6 +44,19 @@ class YoutubeTakeoutActivitySignalServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse omits favorites with an unknown activity date`() { + val zip = createZip( + """ + You liked Video
+ 8 Foo 2026, 19:11:02 CEST
+ """.trimIndent(), + ) + + assertTrue(YoutubeTakeoutActivitySignalService.parse(zip).second.isEmpty()) + Files.deleteIfExists(zip) + } + private fun createZip( html: String = """ Vous vous êtes abonné à Channel Name
diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index 5b960a53..18348b37 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.services.YoutubeTakeoutParserService +import dev.typetype.server.services.YoutubeTakeoutDateParser import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -70,6 +71,76 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse detects spanish takeout dates and playlist names`() { + val zip = Files.createTempFile("yt-takeout-spanish-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/suscripciones/suscripciones.csv")) + out.write("ID de canal,URL del canal,Título del canal\nUC123456789012,https://www.youtube.com/channel/UC123456789012,Canal\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/listas de reproducción.csv")) + out.write("ID de la lista de reproducción,Título de la lista de reproducción\nPL123456789,Importada\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/Videos de Importada.csv")) + out.write("ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000001,2026-01-02T00:00:00Z\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/YouTube y YouTube Music/listas de reproducción/Ver más tarde.csv")) + out.write("ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000002,2026-01-01T00:00:00Z\n".toByteArray()) + out.closeEntry() + out.putNextEntry(ZipEntry("Takeout/Mon actividad/YouTube/watch-history.html")) + out.write("Has visto Watched
16 sept 2026, 18:02:08 CEST
".toByteArray()) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertEquals(1, parsed.subscriptions.size) + assertEquals(1, parsed.playlists.size) + assertEquals(1, parsed.playlistItems["Importada"]?.size) + assertEquals(1, parsed.watchLater.size) + assertEquals(1, parsed.history.size) + assertEquals(1_789_574_528_000L, parsed.history.single().watchedAt) + assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 septiembre 2026, 18:02:08 CEST")) + assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 de septiembre de 2026, 18:02:08 CEST")) + Files.deleteIfExists(zip) + } + + @Test + fun `parse skips history rows with an unknown activity date`() { + val zip = Files.createTempFile("yt-takeout-invalid-date-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/My Activity/YouTube/watch-history.html")) + out.write("You watched Unknown date
16 Foo 2026, 12:00:00 CET
".toByteArray()) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertTrue(parsed.history.isEmpty()) + assertTrue(parsed.history.none { it.watchedAt == 0L }) + Files.deleteIfExists(zip) + } + + @Test + fun `parse accepts a localized history file without an english activity marker`() { + val zip = Files.createTempFile("yt-takeout-localized-history-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.putNextEntry(ZipEntry("Takeout/YouTube/история просмотров/история просмотров.html")) + out.write( + ("Localized title
" + + "Channel
" + + "16. September 2026, 18:02:08 CEST
").toByteArray(), + ) + out.closeEntry() + } + + val parsed = YoutubeTakeoutParserService().parse(zip) + + assertEquals(listOf("localized1"), parsed.history.map { it.url.substringAfter("v=") }) + assertTrue(parsed.history.none { it.watchedAt == 0L }) + Files.deleteIfExists(zip) + } + @Test fun `parse preserves takeout playlist order and added dates`() { val zip = Files.createTempFile("yt-takeout-playlist-order-", ".zip") diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt index 84fb2371..9e750014 100644 --- a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -63,6 +63,51 @@ class YoutubeTakeoutPortabilityAdapterTest { spool.delete() } + @Test + fun `adapter detects spanish playlist paths and activity dates`() { + val archive = directory.resolve("takeout-es.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube y YouTube Music/suscripciones/suscripciones.csv", + "ID de canal,URL del canal,Título del canal\nUC123456789012,https://youtube.com/channel/UC123456789012,Canal\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/catalogo.csv", + "ID de la lista de reproducción,Título de la lista de reproducción\nPL123456789,Importada\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Videos de Importada.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000001,2026-01-02T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Ver más tarde.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000002,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/Mon actividad/YouTube/watch-history.html", + "Has visto Watched
16 sept 2026, 18:02:08 CEST
", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout-es.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + assertEquals(1L, spool.counts()[PortabilityCategory.WATCH_LATER]) + assertEquals(1_789_574_528_000L, (spoolRecord(spool, PortabilityCategory.HISTORY) as PortabilityHistory).watchedAt) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + private fun spoolRecord(spool: PortabilitySpool, category: PortabilityCategory): PortabilityRecord { + var result: PortabilityRecord? = null + spool.forEach(category) { result = it } + return requireNotNull(result) + } + private fun ZipOutputStream.entry(name: String, value: String) { putNextEntry(ZipEntry(name)) write(value.toByteArray()) From e960ab16e3c9bf5737e31c3b8e8bbf213b6af561 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 01:24:45 +0200 Subject: [PATCH 014/137] feat: import YouTube Takeout activity JSON --- .../YoutubeTakeoutJsonEntryReader.kt | 108 +++++++++++ .../YoutubeTakeoutJsonPortabilityReader.kt | 170 ++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt new file mode 100644 index 00000000..f184d338 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt @@ -0,0 +1,108 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken + +internal object YoutubeTakeoutJsonEntryReader { + fun read(parser: JsonParser): YoutubeTakeoutJsonEntry { + var title = "" + var titleUrl = "" + var time = "" + var description = "" + var header = "" + val controls = mutableListOf() + val products = mutableListOf() + val subtitles = mutableListOf() + val details = mutableListOf() + while (parser.nextToken() != JsonToken.END_OBJECT) { + val name = parser.currentName() + when (parser.nextToken()) { + JsonToken.VALUE_STRING, JsonToken.VALUE_NUMBER_INT, JsonToken.VALUE_NUMBER_FLOAT -> when (name) { + "title" -> title = parser.textOrEmpty() + "titleUrl" -> titleUrl = parser.textOrEmpty() + "time", "publishedAt" -> time = parser.textOrEmpty() + "description" -> description = parser.textOrEmpty() + "header" -> header = parser.textOrEmpty() + } + JsonToken.START_OBJECT -> if (name == "snippet") { + val nested = read(parser) + if (title.isBlank()) title = nested.title + if (titleUrl.isBlank()) titleUrl = nested.titleUrl + if (time.isBlank()) time = nested.time + if (description.isBlank()) description = nested.description + if (header.isBlank()) header = nested.header + controls += nested.controls + products += nested.products + subtitles += nested.subtitles + details += nested.details + } else parser.skipChildren() + JsonToken.START_ARRAY -> when (name) { + "activityControls" -> readStrings(parser, controls) + "products" -> readStrings(parser, products) + "subtitles" -> readSubtitles(parser, subtitles) + "details" -> readDetails(parser, details) + else -> parser.skipChildren() + } + else -> parser.skipChildren() + } + } + return YoutubeTakeoutJsonEntry(title, titleUrl, time, description, header, controls, products, subtitles, details) + } + + private fun readStrings(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() == JsonToken.VALUE_STRING) target += parser.textOrEmpty() + else parser.skipChildren() + } + } + + private fun readSubtitles(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + var name = "" + var url = "" + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName() + when (parser.nextToken()) { + JsonToken.VALUE_STRING -> when (field) { + "name" -> name = parser.textOrEmpty() + "url" -> url = parser.textOrEmpty() + } + else -> parser.skipChildren() + } + } + target += YoutubeTakeoutJsonSubtitle(name, url) + } + } + + private fun readDetails(parser: JsonParser, target: MutableList) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + while (parser.nextToken() != JsonToken.END_OBJECT) { + val field = parser.currentName() + if (parser.nextToken() == JsonToken.VALUE_STRING && field == "name") target += parser.textOrEmpty() + else parser.skipChildren() + } + } + } +} + +internal data class YoutubeTakeoutJsonEntry( + val title: String, + val titleUrl: String, + val time: String, + val description: String, + val header: String, + val controls: List, + val products: List, + val subtitles: List, + val details: List, +) + +internal data class YoutubeTakeoutJsonSubtitle(val name: String, val url: String) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt new file mode 100644 index 00000000..5e729111 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt @@ -0,0 +1,170 @@ +package dev.typetype.server.portability + +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonToken +import dev.typetype.server.services.YoutubeTakeoutActivityClassifier +import dev.typetype.server.services.YoutubeTakeoutDateParser +import dev.typetype.server.services.YoutubeTakeoutPathHints +import dev.typetype.server.services.YoutubeTakeoutTextNormalizer +import java.io.InputStream + +internal object YoutubeTakeoutJsonPortabilityReader { + private val videoUrlRegex = Regex( + """(?:https?://|URLs://)(?:www\.|music\.)?youtube\.com/(?:watch\?(?:[^#\s]*&)?v=|shorts/|live/)[A-Za-z0-9_-]{6,}|https?://youtu\.be/[A-Za-z0-9_-]{6,}""", + RegexOption.IGNORE_CASE, + ) + private val channelUrlRegex = Regex( + """https?://(?:www\.)?youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+|c/[A-Za-z0-9._-]+|user/[A-Za-z0-9._-]+)""", + RegexOption.IGNORE_CASE, + ) + private val actionPrefixRegex = Regex( + """^(?:you\s+watched|watched|viewed|you\s+liked|liked|has\s+visto|vous\s+avez\s+regard[eé]|vous\s+avez\s+aime[eé]|te\s+ha\s+gustado|izl[eə]nildi)(?:\s+|$)""", + RegexOption.IGNORE_CASE, + ) + private val actionSuffixRegex = Regex( + """\s+(?:izl[eə]nildi|を視聴しました|を再生しました)\s*$""", + RegexOption.IGNORE_CASE, + ) + + fun isCandidate(path: String): Boolean { + if (!path.endsWith(".json", ignoreCase = true)) return false + val normalized = YoutubeTakeoutTextNormalizer.normalize(path) + val filename = normalized.substringAfterLast('/') + val namedHistory = filename in setOf( + "watch history json", + "watch_history json", + "watch history", + "myactivity json", + "my activity json", + ) || filename.startsWith("myactivity ") || filename.startsWith("my activity ") + return namedHistory || YoutubeTakeoutPathHints.isHistoryEntry(path) || + ("youtube" in normalized && ("activity" in normalized || "history" in normalized)) + } + + fun read(input: InputStream, sink: PortabilityRecordSink) { + PortabilityJsonFactory.createParser(input.buffered()).use { parser -> + read(parser, sink) + } + } + + private fun read(parser: JsonParser, sink: PortabilityRecordSink) { + require(parser.nextToken() == JsonToken.START_ARRAY) { "YouTube Takeout JSON root must be an array" } + var count = 0 + var invalidDates = 0L + var historyRecords = 0L + while (parser.nextToken() != JsonToken.END_ARRAY) { + require(count++ < PortabilityLimits.MAX_CONTAINER_RECORDS) { "YouTube Takeout JSON contains too many records" } + if (parser.currentToken() != JsonToken.START_OBJECT) { + parser.skipChildren() + continue + } + val entry = YoutubeTakeoutJsonEntryReader.read(parser) + if (entry.isAd()) continue + if (!entry.isYoutubeActivity()) continue + val directUrl = findVideoUrl(entry.titleUrl) + val embeddedUrl = findVideoUrl(entry.title) ?: findVideoUrl(entry.description) + val url = directUrl ?: embeddedUrl + val timestamp = entry.time.takeIf(String::isNotBlank)?.let(YoutubeTakeoutDateParser::parseEpochMillis) + val history = url != null && entry.shouldImportHistory(directUrl, embeddedUrl) + val liked = YoutubeTakeoutActivityClassifier.isLikedAction(entry.title) || + entry.controls.any(YoutubeTakeoutActivityClassifier::isLiked) + if (url != null && timestamp != null && history) { + val video = entry.video(url) + sink.write(PortabilityHistory(video, timestamp)) + historyRecords += 1 + if (liked) sink.write(PortabilityFavorite(video, timestamp)) + } else if (url != null && timestamp != null && liked) { + sink.write(PortabilityFavorite(entry.video(url), timestamp)) + } else if (url != null && entry.time.isNotBlank() && history) { + invalidDates += 1 + } + if (YoutubeTakeoutActivityClassifier.isSubscribedAction(entry.title) || + entry.controls.any(YoutubeTakeoutActivityClassifier::isSubscribed) + ) { + entry.channel()?.let { sink.write(PortabilitySubscription(it.url, it.name)) } + } + } + if (historyRecords > 0) sink.markCategory(PortabilityCategory.HISTORY) + if (invalidDates > 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "invalid_takeout_date", + "YouTube Takeout JSON rows with an invalid date were skipped", + invalidDates, + ), + ) + } + } + + private fun YoutubeTakeoutJsonEntry.video(url: String): PortabilityVideo { + val title = actionSuffixRegex.replace(actionPrefixRegex.replace(substringBeforeUrl(), "").trim(), "").trim().ifBlank { + "YouTube video ${youtubeId(url)}" + } + val channel = channel() + return PortabilityVideo( + url = url, + title = title, + thumbnailUrl = "https://i.ytimg.com/vi/${youtubeId(url)}/hqdefault.jpg", + channelName = channel?.name.orEmpty(), + channelUrl = channel?.url.orEmpty(), + ) + } + + private fun YoutubeTakeoutJsonEntry.channel(): YoutubeTakeoutJsonSubtitle? { + subtitles.firstOrNull { channelUrlRegex.containsMatchIn(it.url) }?.let { return it } + val url = channelUrlRegex.find(titleUrl)?.value ?: return null + val name = actionSuffixRegex.replace(actionPrefixRegex.replace(title, "").trim(), "").trim() + return YoutubeTakeoutJsonSubtitle(name, url) + } + + private fun YoutubeTakeoutJsonEntry.substringBeforeUrl(): String = title.replace(videoUrlRegex, "").trim() + + private fun YoutubeTakeoutJsonEntry.isYoutubeActivity(): Boolean { + val markers = buildList { + if (header.isNotBlank()) add(header) + addAll(products) + addAll(controls) + } + return markers.isEmpty() || markers.any { YoutubeTakeoutTextNormalizer.normalize(it).contains("youtube") } + } + + private fun YoutubeTakeoutJsonEntry.shouldImportHistory(directUrl: String?, embeddedUrl: String?): Boolean { + if (isNavigationActivity()) return false + if (YoutubeTakeoutActivityClassifier.isWatchedAction(title)) return true + if (controls.any(YoutubeTakeoutPathHints::isHistoryEntry)) return true + if (YoutubeTakeoutActivityClassifier.isLikedAction(title) || + YoutubeTakeoutActivityClassifier.isSubscribedAction(title) || + controls.any(YoutubeTakeoutActivityClassifier::isLiked) || + controls.any(YoutubeTakeoutActivityClassifier::isSubscribed) + ) return false + return directUrl != null && header.isBlank() && products.isEmpty() && controls.isEmpty() && embeddedUrl == null + } + + private fun YoutubeTakeoutJsonEntry.isNavigationActivity(): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(title) + return normalized.startsWith("visited ") || normalized.startsWith("searched ") || + normalized.startsWith("viewed a post") || normalized.startsWith("viewed ads") + } + + private fun YoutubeTakeoutJsonEntry.isAd(): Boolean = details.any { + val normalized = YoutubeTakeoutTextNormalizer.normalize(it) + normalized in setOf("ads", "advertisement", "advertisements", "annonces", "werbung") || + normalized.contains("google ads") + } || YoutubeTakeoutTextNormalizer.normalize(title).let { + it.startsWith("viewed ads on youtube") || it.startsWith("watched ads on youtube") + } + + private fun findVideoUrl(value: String): String? = videoUrlRegex.find(value)?.value?.let { + when { + it.startsWith("URLs://", ignoreCase = true) -> "https://${it.substringAfter("://")}" + it.startsWith("http://", ignoreCase = true) -> "https://${it.substringAfter("://")}" + else -> it + } + } + + private fun youtubeId(url: String): String = url.substringAfter("v=", "").substringBefore('&').ifBlank { + url.substringAfterLast('/').substringBefore('?') + } + +} From 6d76325bf75b4087c31a1511916c1b2379517a78 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 01:24:53 +0200 Subject: [PATCH 015/137] test: cover YouTube Takeout JSON detection --- .../YoutubeTakeoutPortabilityAdapter.kt | 30 +++++-- .../YoutubeTakeoutActivityClassifier.kt | 20 +++++ .../services/YoutubeTakeoutPathHints.kt | 1 + .../server/PortabilityJsonReproductionTest.kt | 31 +++++++ .../YoutubeTakeoutPortabilityAdapterTest.kt | 89 +++++++++++++++++++ 5 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt index 50e3cbfc..9ff93c08 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt @@ -1,12 +1,13 @@ package dev.typetype.server.portability import java.io.OutputStream +import java.nio.file.Files import java.util.zip.ZipFile class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { override val descriptor = PortabilityAdapterDescriptor( format = PortabilityFormat.YOUTUBE_TAKEOUT, - adapterVersion = 1, + adapterVersion = 2, capabilities = TAKEOUT_CATEGORIES.mapTo(linkedSetOf()) { category -> PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) }, @@ -15,27 +16,46 @@ class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { ) override fun detect(input: PortabilityInput): PortabilityDetection? { - val archive = input.archive ?: return null + val archive = input.archive + if (archive == null) { + if (!YoutubeTakeoutJsonPortabilityReader.isCandidate(input.filename)) return null + return PortabilityDetection( + PortabilityFormat.YOUTUBE_TAKEOUT, + null, + 90, + "YouTube Takeout JSON activity file", + ) + } val names = archive.names.map(String::lowercase) val youtubeFiles = names.count { name -> - "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html")) + "youtube" in name && (name.endsWith(".csv") || name.endsWith(".html") || name.endsWith(".json")) } - if (youtubeFiles == 0) return null + val jsonFiles = names.count { YoutubeTakeoutJsonPortabilityReader.isCandidate(it) } + if (youtubeFiles + jsonFiles == 0) return null val hasTakeoutRoot = names.any { it.startsWith("takeout/") } return PortabilityDetection( PortabilityFormat.YOUTUBE_TAKEOUT, null, if (hasTakeoutRoot) 99 else 90, - "YouTube Takeout CSV or activity files", + "YouTube Takeout CSV, HTML, or JSON activity files", ) } override fun decode(input: PortabilityInput, sink: PortabilityRecordSink) { requireNotNull(detect(input)) { "Unsupported YouTube Takeout archive" } + if (input.archive == null) { + Files.newInputStream(input.path).use { YoutubeTakeoutJsonPortabilityReader.read(it, sink) } + return + } ZipFile(input.path.toFile()).use { zip -> val entries = zip.entries().asSequence().filterNot { it.isDirectory }.toList() YoutubeTakeoutCsvPortabilityReader.readManifests(zip, entries, sink) YoutubeTakeoutHtmlPortabilityReader.read(zip, entries, sink) + entries.filter { YoutubeTakeoutJsonPortabilityReader.isCandidate(it.name) }.forEach { entry -> + zip.getInputStream(entry).use { inputStream -> + YoutubeTakeoutJsonPortabilityReader.read(inputStream, sink) + } + } YoutubeTakeoutCsvPortabilityReader.readContent(zip, entries, sink) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt index d7c91820..e6461848 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt @@ -3,6 +3,7 @@ package dev.typetype.server.services object YoutubeTakeoutActivityClassifier { private val watchedPhrases = setOf( "You watched", + "Viewed", "Vous avez regardé", "Has visto", "Has visto el vídeo", @@ -49,6 +50,12 @@ object YoutubeTakeoutActivityClassifier { "لقد شاهدت", ) + private val watchedSuffixes = setOf( + "izlənildi", + "を視聴しました", + "を再生しました", + ) + private val likedPhrases = setOf( "You liked", "Liked", @@ -124,15 +131,23 @@ object YoutubeTakeoutActivityClassifier { "لقد اشتركت في", ) private val normalizedWatchedPhrases = watchedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) + private val normalizedWatchedSuffixes = watchedSuffixes.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) private val normalizedLikedPhrases = likedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) private val normalizedSubscribedPhrases = subscribedPhrases.mapTo(hashSetOf(), YoutubeTakeoutTextNormalizer::normalize) fun isWatched(value: String): Boolean = containsAny(value, normalizedWatchedPhrases) + fun isWatchedAction(value: String): Boolean = startsWithAny(value, normalizedWatchedPhrases) || + normalizedWatchedSuffixes.any { YoutubeTakeoutTextNormalizer.normalize(value).endsWith(" $it") } + fun isLiked(value: String): Boolean = containsAny(value, normalizedLikedPhrases) + fun isLikedAction(value: String): Boolean = startsWithAny(value, normalizedLikedPhrases) + fun isSubscribed(value: String): Boolean = containsAny(value, normalizedSubscribedPhrases) + fun isSubscribedAction(value: String): Boolean = startsWithAny(value, normalizedSubscribedPhrases) + val watchedPattern: String = pattern(watchedPhrases) val likedPattern: String = pattern(likedPhrases) @@ -144,5 +159,10 @@ object YoutubeTakeoutActivityClassifier { return phrases.any { YoutubeTakeoutTextNormalizer.normalize(it) in normalized } } + private fun startsWithAny(value: String, phrases: Set): Boolean { + val normalized = YoutubeTakeoutTextNormalizer.normalize(value) + return phrases.any { normalized == it || normalized.startsWith("$it ") } + } + private fun pattern(phrases: Set): String = phrases.joinToString("|") { Regex.escape(it) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt index 685757e2..749a8ae6 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt @@ -23,6 +23,7 @@ object YoutubeTakeoutPathHints { "история просмотра", "історія перегляду", "視聴履歴", + "再生履歴", "시청 기록", "观看记录", "觀看記錄", diff --git a/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt b/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt new file mode 100644 index 00000000..bdb14538 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/PortabilityJsonReproductionTest.kt @@ -0,0 +1,31 @@ +package dev.typetype.server + +import dev.typetype.server.portability.PortabilityCategory +import dev.typetype.server.portability.PortabilityInputFactory +import dev.typetype.server.portability.PortabilityLimits +import dev.typetype.server.portability.PortabilitySpool +import dev.typetype.server.portability.YoutubeTakeoutPortabilityAdapter +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable +import java.nio.file.Files +import java.nio.file.Path + +@EnabledIfEnvironmentVariable(named = "PORTABILITY_REPRO_JSON", matches = ".+") +class PortabilityJsonReproductionTest { + @Test + fun `public My Activity JSON reaches preview without account writes`() { + val path = Path.of(System.getenv("PORTABILITY_REPRO_JSON")) + require(Files.size(path) <= PortabilityLimits.MAX_UPLOAD_BYTES) + val input = PortabilityInputFactory.create(path, "watch-history.json", "application/json") + val spool = PortabilitySpool.create(path.parent ?: Path.of("/tmp")) + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + val counts = spool.counts() + assertTrue((counts[PortabilityCategory.HISTORY] ?: 0L) > 0L, counts.toString()) + println("JSON preview counts: ${counts.filterKeys { it in setOf(PortabilityCategory.HISTORY, PortabilityCategory.FAVORITES, PortabilityCategory.SUBSCRIPTIONS) }}") + } finally { + spool.delete() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt index 9e750014..f02ac77a 100644 --- a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server.portability import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir @@ -102,6 +103,94 @@ class YoutubeTakeoutPortabilityAdapterTest { spool.delete() } + @Test + fun `adapter streams My Activity JSON including embedded URLs`() { + val json = directory.resolve("watch-history.json") + Files.writeString( + json, + """[ + {"header":"YouTube","title":"Watched First title","titleUrl":"https://www.youtube.com/watch?v=watched01","subtitles":[{"name":"Channel","url":"https://www.youtube.com/channel/UC123456789012"}],"time":"2026-09-16T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Liked Second title","titleUrl":"https://www.youtube.com/watch?v=liked01","time":"2026-09-15T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Watched URLs://www.youtube.com/watch?v=embedded1","time":"2026-09-14T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"Titre japonais を視聴しました","titleUrl":"https://music.youtube.com/watch?v=japan01","time":"2026-09-13T18:02:08Z","activityControls":["視聴履歴"]}, + {"header":"YouTube","title":"Visited https://youtu.be/visited1","time":"2026-09-12T18:02:08Z","activityControls":["Web & App Activity","YouTube watch history"]}, + {"header":"YouTube","title":"Watched I liked this title","titleUrl":"https://www.youtube.com/watch?v=falsefav1","time":"2026-09-11T18:02:08Z","activityControls":["YouTube watch history"]}, + {"header":"YouTube","title":"You subscribed to Channel","titleUrl":"https://www.youtube.com/channel/UC987654321098","subtitles":[{"name":"Channel","url":"https://www.youtube.com/channel/UC987654321098"}],"time":"2026-09-10T18:02:08Z","activityControls":["YouTube subscriptions"]}, + {"header":"YouTube","title":"You subscribed to Direct channel","titleUrl":"https://www.youtube.com/@directchannel","time":"2026-09-09T18:02:08Z","activityControls":["YouTube subscriptions"]} + ]""".trimIndent(), + ) + val input = PortabilityInputFactory.create(json, json.fileName.toString(), "application/json") + val spool = PortabilitySpool.create(directory) + val adapter = YoutubeTakeoutPortabilityAdapter() + + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(adapter.detect(input)).format) + adapter.decode(input, spool) + + assertEquals(5L, spool.counts()[PortabilityCategory.HISTORY]) + assertEquals(1L, spool.counts()[PortabilityCategory.FAVORITES]) + assertEquals(2L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + val history = mutableListOf() + spool.forEach(PortabilityCategory.HISTORY) { history += it as PortabilityHistory } + assertEquals("https://www.youtube.com/watch?v=embedded1", history[2].video.url) + assertEquals("YouTube video embedded1", history[2].video.title) + assertEquals( + "Titre japonais", + history.first { it.video.url.contains("japan01") }.video.title, + ) + assertTrue(spool.issues().isEmpty()) + spool.delete() + } + + @Test + fun readsLegacySnippetDatesAndSkipsAdRows() { + val json = directory.resolve("legacy-watch-history.json") + Files.writeString( + json, + """[ + {"snippet":{"title":"Watched Legacy title","titleUrl":"https://www.youtube.com/watch?v=legacy01","publishedAt":"2026-09-16T18:02:08Z"}}, + {"title":"Watched advertisement","titleUrl":"https://www.youtube.com/watch?v=adrow01","time":"2026-09-16T18:02:08Z","details":[{"name":"Ads"}]} + ]""".trimIndent(), + ) + val input = PortabilityInputFactory.create(json, "watch-history.json", "application/json") + val spool = PortabilitySpool.create(directory) + try { + assertEquals( + PortabilityFormat.YOUTUBE_TAKEOUT, + requireNotNull(YoutubeTakeoutPortabilityAdapter().detect(input)).format, + ) + assertNotNull( + YoutubeTakeoutPortabilityAdapter().detect( + PortabilityInputFactory.create(json, "再生履歴.json", "application/json"), + ), + ) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + + @Test + fun `adapter reads JSON activity entries from a Takeout archive`() { + val archive = directory.resolve("takeout-json.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/マイ アクティビティ/再生履歴.json", + """[{"header":"YouTube","title":"動画 を視聴しました","titleUrl":"https://www.youtube.com/watch?v=archive01","time":"2026-09-16T18:02:08Z"}]""", + ) + } + val input = PortabilityInputFactory.create(archive, "takeout-json.zip", "application/zip") + val spool = PortabilitySpool.create(directory) + try { + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, requireNotNull(YoutubeTakeoutPortabilityAdapter().detect(input)).format) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + } finally { + spool.delete() + } + } + private fun spoolRecord(spool: PortabilitySpool, category: PortabilityCategory): PortabilityRecord { var result: PortabilityRecord? = null spool.forEach(category) { result = it } From 1b774ec6f78a6eb4b909a521820fa04b7e27055e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 01:30:44 +0200 Subject: [PATCH 016/137] fix: parse localized Takeout day periods --- .../services/YoutubeTakeoutDateParser.kt | 16 ++++++++++++++ .../YoutubeTakeoutDateParserLocaleTest.kt | 21 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt index 44ebd295..a4102ee9 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -31,8 +31,10 @@ object YoutubeTakeoutDateParser { "M/d/uuuu, HH:mm:ss z", "yyyy年M月d日 HH:mm:ss z", "yyyy年M月d日 H:mm:ss z", + "yyyy年M月d日 a h:mm:ss z", "yyyy년 M월 d일 HH:mm:ss z", "yyyy년 M월 d일 H:mm:ss z", + "yyyy년 M월 d일 a h:mm:ss z", "d M月 yyyy, HH:mm:ss z", "d M月 yyyy HH:mm:ss z", "d M월 yyyy, HH:mm:ss z", @@ -82,6 +84,7 @@ object YoutubeTakeoutDateParser { private fun normalizeActivityText(value: String): String = value .replace('\u060C', ',') .replace('\uFF0C', ',') + .replace(DAY_PERIOD_REGEX) { DAY_PERIODS[it.value] ?: it.value } .replace(ACTIVITY_CONNECTOR_REGEX, " ") .replace(ACTIVITY_SPACES_REGEX, " ") .trim() @@ -146,4 +149,17 @@ object YoutubeTakeoutDateParser { private val MONTH_TOKEN_REGEX = Regex("[\\p{L}\\p{M}][\\p{L}\\p{M}.]*") private val ACTIVITY_CONNECTOR_REGEX = Regex("\\s+(?:de|del)\\s+", RegexOption.IGNORE_CASE) private val ACTIVITY_SPACES_REGEX = Regex("\\s+") + private val DAY_PERIOD_REGEX = Regex("凌晨|早上|上午|中午|下午|晚上|午前|午後|오전|오후") + private val DAY_PERIODS = mapOf( + "凌晨" to "AM ", + "早上" to "AM ", + "上午" to "AM ", + "中午" to "PM ", + "下午" to "PM ", + "晚上" to "PM ", + "午前" to "AM ", + "午後" to "PM ", + "오전" to "AM ", + "오후" to "PM ", + ) } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt new file mode 100644 index 00000000..779e437f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt @@ -0,0 +1,21 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutDateParser +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test + +class YoutubeTakeoutDateParserLocaleTest { + @Test + fun `parses CLDR periods used by Chinese Japanese and Korean Takeout exports`() { + assertEquals( + 1_788_576_327_000L, + YoutubeTakeoutDateParser.parseEpochMillis("2026年9月4日 晚上9:45:27 CST"), + ) + assertEquals( + 1_788_525_927_000L, + YoutubeTakeoutDateParser.parseEpochMillis("2026年9月4日 午後9:45:27 JST"), + ) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026년 9월 4일 오후 9:45:27 KST")) + } +} From 422df6bd554cf7b9f9c590a93ddb63f690bc7fbb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 02:05:17 +0200 Subject: [PATCH 017/137] fix: broaden localized Takeout imports and progress --- .../server/portability/PortabilityEngine.kt | 1 + .../server/portability/PortabilityProgress.kt | 4 ++ .../YoutubeTakeoutActivitySignalService.kt | 2 +- .../services/YoutubeTakeoutDateParser.kt | 41 ++++++++++++++++- .../services/YoutubeTakeoutHistoryParser.kt | 4 +- .../services/YoutubeTakeoutPathHints.kt | 14 +++++- .../services/YoutubeTakeoutSchemaHints.kt | 3 +- .../services/YoutubeTakeoutSchemaTerms.kt | 11 +++-- .../server/services/YoutubeTypeTypeMapper.kt | 2 + .../YoutubeTakeoutDateParserLocaleTest.kt | 8 ++++ .../server/YoutubeTakeoutParserServiceTest.kt | 18 ++++++++ .../server/YoutubeTakeoutPathHintsTest.kt | 14 ++++++ .../portability/PortabilityProgressTest.kt | 46 +++++++++++++++++++ 13 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index 4f788a5e..bb893493 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -132,6 +132,7 @@ class PortabilityEngine internal constructor( PortabilityProgressPhase.APPLYING, PortabilityProgressUnit.RECORDS, total, + interval = portabilityProgressInterval(total), ) val result = dataPort.import( job.ownerId, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt index 265850d2..5f92e28b 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt +++ b/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -41,6 +41,10 @@ internal class PortabilityProgressReporter( } } +internal fun portabilityProgressInterval(total: Long?): Long = total + ?.let { (it / 100L).coerceIn(1L, 100L) } + ?: 100L + internal class ProgressRecordSink( private val delegate: PortabilityRecordSink, private val progress: PortabilityProgressReporter, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt index cbbabeaf..4a520859 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt @@ -8,7 +8,7 @@ import java.util.zip.ZipFile object YoutubeTakeoutActivitySignalService { private val activityLinkRegex = Regex("""]*>([^<]*)""", RegexOption.IGNORE_CASE) private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) - private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") + private val watchUrlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/|live/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val channelUrlRegex = Regex("""https?://www\.youtube\.com/(?:channel/[A-Za-z0-9_-]+|@[A-Za-z0-9._-]+)""") private val spacesRegex = Regex("""\s+""") diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt index a4102ee9..25953258 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -29,6 +29,20 @@ object YoutubeTakeoutDateParser { "MMMM d, yyyy HH:mm:ss z", "d/M/uuuu, HH:mm:ss z", "M/d/uuuu, HH:mm:ss z", + "d-M-uuuu, HH:mm:ss z", + "M-d-uuuu, HH:mm:ss z", + "d.M.uuuu, HH:mm:ss z", + "M.d.uuuu, HH:mm:ss z", + "yyyy/M/d, HH:mm:ss z", + "yyyy/M/d, H:mm:ss z", + "yyyy/M/d HH:mm:ss z", + "yyyy/M/d H:mm:ss z", + "yyyy-MM-dd, HH:mm:ss z", + "yyyy-MM-dd, H:mm:ss z", + "yyyy-MM-dd HH:mm:ss z", + "yyyy-MM-dd H:mm:ss z", + "yyyy.MM.dd, HH:mm:ss z", + "yyyy.MM.dd HH:mm:ss z", "yyyy年M月d日 HH:mm:ss z", "yyyy年M月d日 H:mm:ss z", "yyyy年M月d日 a h:mm:ss z", @@ -94,7 +108,8 @@ object YoutubeTakeoutDateParser { }.getOrNull() private fun canonicalizeMonths(value: String): String = MONTH_TOKEN_REGEX.replace(value) { match -> - monthAliases[normalizeMonth(match.value)] ?: match.value + val token = normalizeMonth(match.value) + if (token in DAY_PERIOD_TOKENS) match.value else monthAliases[token] ?: match.value } private fun buildMonthAliases(): Map { @@ -149,17 +164,39 @@ object YoutubeTakeoutDateParser { private val MONTH_TOKEN_REGEX = Regex("[\\p{L}\\p{M}][\\p{L}\\p{M}.]*") private val ACTIVITY_CONNECTOR_REGEX = Regex("\\s+(?:de|del)\\s+", RegexOption.IGNORE_CASE) private val ACTIVITY_SPACES_REGEX = Regex("\\s+") - private val DAY_PERIOD_REGEX = Regex("凌晨|早上|上午|中午|下午|晚上|午前|午後|오전|오후") + private val DAY_PERIOD_REGEX = Regex( + "凌晨|清晨|早晨|早上|上午|中午|下午|晚上|半夜|午前|午後|오전|오후|" + + "صباحًا|صباحا|صباح|مساءً|مساءا|مساء|قبل الظهر|بعد الظهر|" + + "पूर्वाह्न|अपराह्न|सुबह|दोपहर|शाम|रात", + ) private val DAY_PERIODS = mapOf( "凌晨" to "AM ", + "清晨" to "AM ", + "早晨" to "AM ", "早上" to "AM ", "上午" to "AM ", "中午" to "PM ", "下午" to "PM ", "晚上" to "PM ", + "半夜" to "AM ", "午前" to "AM ", "午後" to "PM ", "오전" to "AM ", "오후" to "PM ", + "صباحًا" to "AM ", + "صباحا" to "AM ", + "صباح" to "AM ", + "مساءً" to "PM ", + "مساءا" to "PM ", + "مساء" to "PM ", + "قبل الظهر" to "AM ", + "بعد الظهر" to "PM ", + "पूर्वाह्न" to "AM ", + "अपराह्न" to "PM ", + "सुबह" to "AM ", + "दोपहर" to "PM ", + "शाम" to "PM ", + "रात" to "PM ", ) + private val DAY_PERIOD_TOKENS = setOf("am", "pm") } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index a01390f4..0895a5e4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -4,11 +4,11 @@ import dev.typetype.server.models.HistoryItem object YoutubeTakeoutHistoryParser { private val videoLinkRegex = Regex( - """]*>([^<]*)""", + """]*>([^<]*)""", RegexOption.IGNORE_CASE, ) private val rowTailRegex = Regex("""\s*
\s*(?:([^<]*)
\s*)?([^<]*)
""", RegexOption.IGNORE_CASE) - private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") + private val urlRegex = Regex("""https?://(?:www\.)?(?:youtube\.com/(?:watch\?v=|shorts/|live/)|youtu\.be/)[A-Za-z0-9_-]{6,}""") private val tagRegex = Regex("<[^>]+>") private val spacesRegex = Regex("\\s+") diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt index 749a8ae6..9ec0ac7b 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt @@ -6,19 +6,29 @@ object YoutubeTakeoutPathHints { fun isHistoryEntry(path: String): Boolean { val normalized = YoutubeTakeoutTextNormalizer.normalize(path) - return HISTORY_MARKERS.any { it in normalized } + return NORMALIZED_HISTORY_MARKERS.any { it in normalized } } private val HISTORY_MARKERS = setOf( "watch history", "historique", + "historique des videos regardees", "historico", + "historico de visualizacao", + "historico de exibicao", "historial", + "historial de reproduccion", + "historial de visualizacion", "cronologia", + "cronologia de reproduccion", "verlauf", + "wiedergabeverlauf", "kijkgeschiedenis", "historia ogladania", "izleme gecmisi", + "lich su xem", + "ประวัติการดู", + "ประวัติการรับชม", "история просмотров", "история просмотра", "історія перегляду", @@ -31,4 +41,6 @@ object YoutubeTakeoutPathHints { "تاریخچه تماشا", "देखने का इतिहास", ) + + private val NORMALIZED_HISTORY_MARKERS = HISTORY_MARKERS.map(YoutubeTakeoutTextNormalizer::normalize) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt index 8d4e67b2..54e25b94 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt @@ -81,7 +81,8 @@ object YoutubeTakeoutSchemaHints { fun containsWatchUrl(value: String): Boolean { val normalized = value.lowercase() - return "youtube.com/watch?v=" in normalized || "youtube.com/shorts/" in normalized || "youtu.be/" in normalized + return "youtube.com/watch?v=" in normalized || "youtube.com/shorts/" in normalized || + "youtube.com/live/" in normalized || "youtu.be/" in normalized } fun normalize(value: String): String = YoutubeTakeoutTextNormalizer.normalize(value) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt index bbeea247..699c6ac1 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt @@ -15,29 +15,32 @@ internal object YoutubeTakeoutSchemaTerms { "wiedergabeliste", "wiedergabelisten", "afspeellijst", "afspeellijsten", "список воспроизведения", "списки воспроизведения", "плейлист", "плейлисты", "再生リスト", "재생목록", "播放列表", "播放清單", "قائمة التشغيل", "قوائم التشغيل", "प्लेलिस्ट", "प्लेलिस्टें", - "λίστες αναπαραγωγής", + "λίστα αναπαραγωγής", "λίστες αναπαραγωγής", "danh sach phat", "รายการเล่น", "เพลย์ลิสต์", + "רשימת השמעה", "רשימות השמעה", ) val channelWords = setOf( "channel", "chaine", "canal", "canale", "kanal", "kanaal", "канал", "канали", "kanał", - "チャンネル", "채널", "频道", "頻道", "قناة", "चैनल", "κανάλι", + "チャンネル", "채널", "频道", "頻道", "قناة", "चैनल", "κανάλι", "kenh", "ช่อง", "ערוץ", ) val videoWords = setOf( "video", "videos", "vidéo", "vidéos", "vídeo", "vídeos", "動画", "동영상", "비디오", "видео", - "视频", "影片", "film", "filmy", "فيديو", "فيديوهات", "वीडियो", "βίντεο", "วิดีโอ", + "视频", "影片", "film", "filmy", "فيديو", "فيديوهات", "वीडियो", "βίντεο", "วิดีโอ", "ভিডিও", "סרטון", ) val subscriptionWords = setOf( "subscriptions", "suscripciones", "inscricoes", "inscrições", "iscrizioni", "abbonamenti", "abos", "abonelikler", "abonnements", "abonnementen", "abonnierte kanäle", "subskrypcje", "подписки", "підписки", "登録チャンネル", "チャンネル登録", "구독", "订阅", "訂閱", "الاشتراكات", + "การสมัครรับข้อมูล", "การติดตาม", "מינויים", "সাবস্ক্রিপশন", "kenh da dang ky", ) val playlistManifestNames = setOf( "playlists", "oynatma listeleri", "oynatma listesi", "listas de reproduccion", "listes de lecture", "liste di riproduzione", "listas de reproducao", "wiedergabelisten", "wiedergabeliste", "afspeellijsten", "afspeellijst", "списки воспроизведения", "список воспроизведения", "再生リスト", - "재생목록", "播放列表", "播放清單", "قوائم التشغيل", "قائمة التشغيل", + "재생목록", "播放列表", "播放清單", "قوائم التشغيل", "قائمة التشغيل", "danh sach phat", + "รายการเล่น", "เพลย์ลิสต์", "רשימת השמעה", "רשימות השמעה", ).mapTo(mutableSetOf(), YoutubeTakeoutTextNormalizer::normalize) } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt index e2082b0a..ca74ad09 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt @@ -33,6 +33,7 @@ object YoutubeTypeTypeMapper { val trimmed = url.trim() WATCH_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } SHORTS_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } + LIVE_ID_REGEX.find(trimmed)?.let { return it.groupValues[1] } SHORT_URL_REGEX.find(trimmed)?.let { return it.groupValues[1] } return trimmed.takeIf { RAW_ID_REGEX.matches(it) } } @@ -43,6 +44,7 @@ object YoutubeTypeTypeMapper { private val WATCH_ID_REGEX = Regex("""[?&]v=([A-Za-z0-9_-]{6,})""") private val SHORTS_ID_REGEX = Regex("""youtube\.com/shorts/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) + private val LIVE_ID_REGEX = Regex("""youtube\.com/live/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) private val SHORT_URL_REGEX = Regex("""youtu\.be/([A-Za-z0-9_-]{6,})""", RegexOption.IGNORE_CASE) private val RAW_ID_REGEX = Regex("""^[A-Za-z0-9_-]{6,}$""") } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt index 779e437f..8f75bb3e 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutDateParserLocaleTest.kt @@ -18,4 +18,12 @@ class YoutubeTakeoutDateParserLocaleTest { ) assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026년 9월 4일 오후 9:45:27 KST")) } + + @Test + fun `parses year first and localized day period variants`() { + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026/09/16 18:02:08 CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("2026-09-16, 18:02:08 CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("4 سبتمبر 2026, 9:45:27 مساءً CEST")) + assertNotNull(YoutubeTakeoutDateParser.parseEpochMillis("4 सितंबर 2026, 9:45:27 शाम CEST")) + } } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index 18348b37..0fcb97db 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -249,6 +249,24 @@ class YoutubeTakeoutParserServiceTest { Files.deleteIfExists(zip) } + @Test + fun `parse preserves live video history urls`() { + val zip = Files.createTempFile("yt-takeout-live-history-", ".zip") + ZipOutputStream(Files.newOutputStream(zip)).use { out -> + out.writeEntry( + "Takeout/My Activity/YouTube/watch-history.html", + "You watched Live stream
" + + "1 Jan 2026, 12:00:00 CET
", + ) + } + + val history = YoutubeTakeoutParserService().parse(zip).history + + assertEquals("https://www.youtube.com/live/live000001", history.single().url) + assertEquals("https://i.ytimg.com/vi/live000001/hqdefault.jpg", history.single().thumbnail) + Files.deleteIfExists(zip) + } + private fun createZip(): Path { val zip = Files.createTempFile("yt-takeout-parser-", ".zip") ZipOutputStream(Files.newOutputStream(zip)).use { out -> diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt new file mode 100644 index 00000000..13e46cc2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt @@ -0,0 +1,14 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutPathHints +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class YoutubeTakeoutPathHintsTest { + @Test + fun `recognizes localized history paths after accent normalization`() { + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/lịch sử xem.html")) + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/ประวัติการรับชม.html")) + assertTrue(YoutubeTakeoutPathHints.isHistoryEntry("Takeout/YouTube/Wiedergabeverlauf.html")) + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt b/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt new file mode 100644 index 00000000..2e9370c5 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class PortabilityProgressTest { + @TempDir + lateinit var directory: Path + + @Test + fun `progress interval publishes every record for small imports`() { + val job = job() + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.RECORDS, + total = 47L, + interval = portabilityProgressInterval(47L), + ) + + progress.add() + + assertEquals(1L, job.snapshot().progress?.processed) + assertEquals(47L, job.snapshot().progress?.total) + } + + @Test + fun `progress interval stays bounded for large imports`() { + assertEquals(1L, portabilityProgressInterval(0L)) + assertEquals(1L, portabilityProgressInterval(47L)) + assertEquals(8L, portabilityProgressInterval(839L)) + assertEquals(100L, portabilityProgressInterval(42_622L)) + assertEquals(100L, portabilityProgressInterval(null)) + } + + private fun job() = PortabilityJob( + id = "progress-test", + ownerId = "owner", + kind = PortabilityJobKind.IMPORT, + directory = directory, + requestId = null, + clock = System::currentTimeMillis, + ) +} From 37333a55e188b3d540568e93762361256dd98503 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 09:02:36 +0200 Subject: [PATCH 018/137] fix: detect localized YouTube Takeout schemas --- .../YoutubeTakeoutCsvPortabilityReader.kt | 56 +++----- .../YoutubeTakeoutCsvSchemaDetector.kt | 133 ++++++++++++++++++ .../YoutubeTakeoutPortabilityAdapter.kt | 5 +- .../services/YoutubeTakeoutRowParser.kt | 2 +- .../services/YoutubeTakeoutSchemaHints.kt | 2 + .../services/YoutubeTakeoutZipScanner.kt | 25 +++- .../YoutubeTakeoutZipScannerSchemaTest.kt | 46 ++++++ .../YoutubeTakeoutCsvSchemaDetectorTest.kt | 55 ++++++++ 8 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt create mode 100644 src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index fe07606e..f514c62d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -13,10 +13,11 @@ import java.util.zip.ZipFile internal object YoutubeTakeoutCsvPortabilityReader { private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" - fun readManifests(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - csvEntries(entries).filter { isPlaylistManifest(zip, it) }.forEach { entry -> + fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { + val classified = YoutubeTakeoutCsvSchemaDetector.classify(zip, entries) + classified.filter { it.kind == YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST }.forEach { classifiedEntry -> sink.markCategory(PortabilityCategory.PLAYLISTS) - forEachRow(zip, entry) { header, row -> + forEachRow(zip, classifiedEntry.entry) { header, row -> val playlist = YoutubeTakeoutRowParser.parsePlaylist(header, row) ?: return@forEachRow indexPlaylist(playlist, sink) if (!isSystemPlaylist(playlist)) { @@ -24,12 +25,20 @@ internal object YoutubeTakeoutCsvPortabilityReader { } } } - } - - fun readContent(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { - val csv = csvEntries(entries).filterNot { isPlaylistManifest(zip, it) }.toList() - csv.filter(::isSubscriptionFile).forEach { readSubscriptions(zip, it, sink) } - csv.filter(::isPlaylistContent).forEach { readPlaylistItems(zip, it, sink) } + classified.filter { it.kind == YoutubeTakeoutCsvKind.SUBSCRIPTIONS } + .forEach { readSubscriptions(zip, it.entry, sink) } + classified.filter { it.kind == YoutubeTakeoutCsvKind.PLAYLIST_CONTENT } + .forEach { readPlaylistItems(zip, it.entry, sink) } + classified.filter { it.kind == YoutubeTakeoutCsvKind.OTHER && it.maybePortable } + .forEach { classifiedEntry -> + sink.issue( + PortabilityIssue( + category = null, + code = "unsupported_takeout_csv", + message = "A YouTube Takeout CSV file could not be classified and was skipped: ${classifiedEntry.entry.name}", + ), + ) + } } private fun readSubscriptions(zip: ZipFile, entry: ZipEntry, sink: PortabilityRecordSink) { @@ -95,39 +104,10 @@ internal object YoutubeTakeoutCsvPortabilityReader { } } - private fun csvEntries(entries: List) = entries.asSequence().filter { - it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() - } - - private fun isPlaylistManifest(zip: ZipFile, entry: ZipEntry): Boolean { - if (YoutubeTakeoutSchemaHints.isPlaylistManifestName(fileStem(entry))) return true - if (!YoutubeTakeoutSchemaHints.isPlaylistText(entry.name)) return false - var manifest = false - forEachRow(zip, entry) { header, _ -> - manifest = manifest || ( - header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) && - header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) && - header.none(YoutubeTakeoutSchemaHints::isVideoIdHeader) - ) - } - return manifest - } - - private fun isSubscriptionFile(entry: ZipEntry): Boolean = - YoutubeTakeoutSchemaHints.isSubscriptionText(fileStem(entry)) - - private fun isPlaylistContent(entry: ZipEntry): Boolean { - return YoutubeTakeoutSchemaHints.isPlaylistText(entry.name) - } - private fun playlistKeyFromPath(path: String): String? = path.substringAfterLast('/').substringBeforeLast('.') .takeUnless { YoutubeTakeoutSchemaHints.normalize(it) == "playlist items" } ?.let { YoutubeTakeoutSystemPlaylist.canonicalKey(it) ?: it } - private fun fileStem(entry: ZipEntry) = YoutubeTakeoutSchemaHints.normalize( - entry.name.substringAfterLast('/').substringBeforeLast('.'), - ) - private fun isSystemPlaylist(item: PlaylistItem) = YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt new file mode 100644 index 00000000..edca04f7 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt @@ -0,0 +1,133 @@ +package dev.typetype.server.portability + +import dev.typetype.server.services.YoutubeTakeoutCsvReader +import dev.typetype.server.services.YoutubeTakeoutDateParser +import dev.typetype.server.services.YoutubeTakeoutSchemaHints +import dev.typetype.server.services.YoutubeTakeoutTextNormalizer +import java.io.BufferedReader +import java.io.InputStreamReader +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +internal enum class YoutubeTakeoutCsvKind { + PLAYLIST_MANIFEST, + SUBSCRIPTIONS, + PLAYLIST_CONTENT, + OTHER, +} + +internal data class YoutubeTakeoutCsvClassification( + val entry: ZipEntry, + val kind: YoutubeTakeoutCsvKind, + val maybePortable: Boolean, +) + +internal object YoutubeTakeoutCsvSchemaDetector { + fun classify(zip: ZipFile, entries: List): List = entries.asSequence() + .filter { it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() } + .map { entry -> classify(zip, entry) } + .toList() + + private fun classify(zip: ZipFile, entry: ZipEntry): YoutubeTakeoutCsvClassification { + val stem = fileStem(entry) + val path = YoutubeTakeoutSchemaHints.normalize(entry.name) + if (YoutubeTakeoutSchemaHints.isPlaylistManifestName(stem)) { + return YoutubeTakeoutCsvClassification(entry, YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST, maybePortable = false) + } + if (YoutubeTakeoutSchemaHints.isSubscriptionText(stem)) { + return YoutubeTakeoutCsvClassification(entry, YoutubeTakeoutCsvKind.SUBSCRIPTIONS, maybePortable = false) + } + if (YoutubeTakeoutSchemaHints.isPlaylistText(path)) { + val shape = inspect(zip, entry) + val kind = if (shape.isManifest()) YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST else YoutubeTakeoutCsvKind.PLAYLIST_CONTENT + return YoutubeTakeoutCsvClassification(entry, kind, maybePortable = false) + } + val shape = inspect(zip, entry) + val kind = when { + shape.isManifest() -> YoutubeTakeoutCsvKind.PLAYLIST_MANIFEST + shape.isSubscription() -> YoutubeTakeoutCsvKind.SUBSCRIPTIONS + shape.isContent() -> YoutubeTakeoutCsvKind.PLAYLIST_CONTENT + else -> YoutubeTakeoutCsvKind.OTHER + } + return YoutubeTakeoutCsvClassification(entry, kind, shape.maybePortable()) + } + + private fun inspect(zip: ZipFile, entry: ZipEntry): CsvShape { + var header = emptyList() + val rows = ArrayList>(SAMPLE_ROWS) + zip.getInputStream(entry).use { input -> + val reader = BufferedReader(InputStreamReader(input, Charsets.UTF_8)) + YoutubeTakeoutCsvReader.forEach(reader, { header = it }) { row -> + if (rows.size < SAMPLE_ROWS) rows += row + } + } + return CsvShape(header, rows) + } + + private fun fileStem(entry: ZipEntry): String = YoutubeTakeoutSchemaHints.normalize( + entry.name.substringAfterLast('/').substringBeforeLast('.'), + ) + + private data class CsvShape( + val header: List, + val rows: List>, + ) { + private val hasChannelId by lazy { + header.any(YoutubeTakeoutSchemaHints::isChannelIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeChannelId) + } + private val hasChannelUrl by lazy { + header.any(YoutubeTakeoutSchemaHints::isChannelUrlHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::containsChannelUrl) + } + private val hasVideoId by lazy { + header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(::isLikelyVideoId) + } + private val hasPlaylistId by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikePlaylistId) + } + private val hasPlaylistKey by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || + header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) + } + private val hasAddedAt by lazy { + header.any(YoutubeTakeoutSchemaHints::isPlaylistItemAddedAtHeader) || rows.hasValue { + YoutubeTakeoutDateParser.parseEpochMillis(it) != null + } + } + + fun isSubscription(): Boolean = hasChannelId && hasChannelUrl && !hasVideoId && !hasPlaylistId + + fun isManifest(): Boolean = hasPlaylistId && !hasVideoId && + (header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) || rows.any { row -> row.any(::isTextValue) }) + + fun isContent(): Boolean { + if (!hasVideoId || hasChannelId) return false + return hasPlaylistKey || header.size <= 3 && (hasAddedAt || rows.isNotEmpty()) + } + + fun maybePortable(): Boolean = isSubscription() || isManifest() || isContent() || + hasChannelUrl || hasPlaylistId || hasVideoId && !hasChannelId && header.size <= 3 + + private fun isTextValue(value: String): Boolean { + val trimmed = value.trim() + if (trimmed.isBlank() || YoutubeTakeoutSchemaHints.looksLikeChannelId(trimmed) || + YoutubeTakeoutSchemaHints.looksLikePlaylistId(trimmed) || + isLikelyVideoId(trimmed) || + YoutubeTakeoutSchemaHints.containsChannelUrl(trimmed) || + YoutubeTakeoutSchemaHints.containsWatchUrl(trimmed) || + YoutubeTakeoutDateParser.parseEpochMillis(trimmed) != null + ) return false + return YoutubeTakeoutTextNormalizer.normalize(trimmed) !in NON_TEXT_VALUES + } + + private fun isLikelyVideoId(value: String): Boolean = + YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(value) + + private fun List>.hasValue(predicate: (String) -> Boolean): Boolean = any { row -> row.any(predicate) } + } + + private val NON_TEXT_VALUES = setOf( + "true", "false", "yes", "no", "oui", "non", "faux", "vrai", "public", "private", "prive", + "publique", "privee", "unlisted", "non repertorie", + ) + private const val SAMPLE_ROWS = 64 +} diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt index 9ff93c08..3cd932f5 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt @@ -7,7 +7,7 @@ import java.util.zip.ZipFile class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { override val descriptor = PortabilityAdapterDescriptor( format = PortabilityFormat.YOUTUBE_TAKEOUT, - adapterVersion = 2, + adapterVersion = 3, capabilities = TAKEOUT_CATEGORIES.mapTo(linkedSetOf()) { category -> PortabilityCapability(category, setOf(PortabilityDirection.IMPORT), PortabilityFidelity.COMPLETE) }, @@ -49,14 +49,13 @@ class YoutubeTakeoutPortabilityAdapter : PortabilityAdapter { } ZipFile(input.path.toFile()).use { zip -> val entries = zip.entries().asSequence().filterNot { it.isDirectory }.toList() - YoutubeTakeoutCsvPortabilityReader.readManifests(zip, entries, sink) + YoutubeTakeoutCsvPortabilityReader.read(zip, entries, sink) YoutubeTakeoutHtmlPortabilityReader.read(zip, entries, sink) entries.filter { YoutubeTakeoutJsonPortabilityReader.isCandidate(it.name) }.forEach { entry -> zip.getInputStream(entry).use { inputStream -> YoutubeTakeoutJsonPortabilityReader.read(inputStream, sink) } } - YoutubeTakeoutCsvPortabilityReader.readContent(zip, entries, sink) } } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt index d9cf8d5f..98ab2f14 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt @@ -88,7 +88,7 @@ object YoutubeTakeoutRowParser { value.isNotBlank() && !YoutubeTakeoutSchemaHints.looksLikeChannelId(value) && !YoutubeTakeoutSchemaHints.looksLikePlaylistId(value) && - !YoutubeTakeoutSchemaHints.looksLikeVideoId(value) && + !YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(value) && !YoutubeTakeoutSchemaHints.containsChannelUrl(value) && !YoutubeTakeoutSchemaHints.containsWatchUrl(value) && !isMetadataValue(value) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt index 54e25b94..1d758272 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt @@ -75,6 +75,8 @@ object YoutubeTakeoutSchemaHints { return videoIdRegex.matches(trimmed) && !looksLikeChannelId(trimmed) && !looksLikePlaylistId(trimmed) } + fun looksLikeLikelyVideoId(value: String): Boolean = value.trim().length >= 10 && looksLikeVideoId(value) + fun looksLikePlaylistId(value: String): Boolean = playlistIdRegex.matches(value.trim()) fun containsChannelUrl(value: String): Boolean = channelUrlRegex.containsMatchIn(value) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt index d940111f..66cdbace 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt @@ -29,7 +29,7 @@ object YoutubeTakeoutZipScanner { if (subscriptionsHeader.isEmpty()) subscriptionsHeader = header subscriptionsRows += rows } - isPlaylistsHeader(entry.name, header, rows) -> { + isPlaylistsHeader(header, rows) -> { if (playlistsHeader.isEmpty()) playlistsHeader = header playlistsRows += rows } @@ -66,10 +66,11 @@ object YoutubeTakeoutZipScanner { return hasId && (hasUrl || (hasTitle && isSubscriptionPath(path))) } - private fun isPlaylistsHeader(path: String, header: List, rows: List>): Boolean { + private fun isPlaylistsHeader(header: List, rows: List>): Boolean { val hasId = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikePlaylistId) - val hasTitle = header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) - return hasId && hasTitle && (isMainPlaylistsFile(path) || isPlaylistPath(path)) + val hasTitle = header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) || rows.hasValue(::looksLikeTextValue) + val hasVideo = header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeLikelyVideoId) + return hasId && hasTitle && !hasVideo } private fun isPlaylistItemsEntry(path: String, header: List, rows: List>): Boolean { @@ -77,7 +78,10 @@ object YoutubeTakeoutZipScanner { val hasVideo = header.any(YoutubeTakeoutSchemaHints::isVideoIdHeader) || rows.hasValue(YoutubeTakeoutSchemaHints::looksLikeVideoId) val hasPlaylistKey = header.any(YoutubeTakeoutSchemaHints::isPlaylistIdHeader) || header.any(YoutubeTakeoutSchemaHints::isPlaylistTitleHeader) - return hasVideo && (isPlaylistPath(path) || isPlaylistItemsFile(path) || hasPlaylistKey) + val hasAddedAt = header.any(YoutubeTakeoutSchemaHints::isPlaylistItemAddedAtHeader) || + rows.hasValue { YoutubeTakeoutDateParser.parseEpochMillis(it) != null } + val compactPlaylistRows = header.size <= 3 && hasAddedAt + return hasVideo && (isPlaylistPath(path) || isPlaylistItemsFile(path) || hasPlaylistKey || compactPlaylistRows) } private fun extractPlaylistSourceKey(path: String, header: List): String? { @@ -102,4 +106,15 @@ object YoutubeTakeoutZipScanner { private fun List>.hasValue(predicate: (String) -> Boolean): Boolean = any { row -> row.any(predicate) } + private fun looksLikeTextValue(value: String): Boolean { + val trimmed = value.trim() + return trimmed.isNotBlank() && + !YoutubeTakeoutSchemaHints.looksLikeChannelId(trimmed) && + !YoutubeTakeoutSchemaHints.looksLikePlaylistId(trimmed) && + !YoutubeTakeoutSchemaHints.looksLikeLikelyVideoId(trimmed) && + !YoutubeTakeoutSchemaHints.containsChannelUrl(trimmed) && + !YoutubeTakeoutSchemaHints.containsWatchUrl(trimmed) && + YoutubeTakeoutDateParser.parseEpochMillis(trimmed) == null + } + } diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt new file mode 100644 index 00000000..a5909279 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutZipScannerSchemaTest.kt @@ -0,0 +1,46 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutParserService +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutZipScannerSchemaTest { + @Test + fun `parse recognizes csv schemas without translated paths`() { + val archive = Files.createTempFile("yt-takeout-structure-", ".zip") + try { + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/section/data-a.csv", + "column-a,column-b,column-c\nUC1234567890,Channel,https://www.youtube.com/channel/UC1234567890\n", + ) + output.entry( + "Takeout/YouTube/section/data-b.csv", + "column-a,column-b\nPL123456,My list\n", + ) + output.entry( + "Takeout/YouTube/section/My list.csv", + "column-a,column-b,column-c\nvideo000001,Video,2026-09-16T18:02:08Z\n", + ) + } + + val parsed = YoutubeTakeoutParserService().parse(archive) + + assertEquals(1, parsed.subscriptions.size) + assertEquals(1, parsed.playlists.size) + assertEquals(1, parsed.playlistItems["My list"]?.size) + } finally { + Files.deleteIfExists(archive) + } + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt new file mode 100644 index 00000000..c076429f --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt @@ -0,0 +1,55 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutCsvSchemaDetectorTest { + @TempDir + lateinit var directory: Path + + @Test + fun `classifies unknown file names from ids urls and row shape`() { + val archive = directory.resolve("takeout-unknown-language.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/section/data-a.csv", + "column-a,column-b,column-c\nUC123456789012,Channel,https://www.youtube.com/channel/UC123456789012\n", + ) + output.entry( + "Takeout/YouTube/section/data-b.csv", + "column-a,column-b\nPL123456789,My list\n", + ) + output.entry( + "Takeout/YouTube/My list.csv", + "column-a,column-b,column-c\nvideo000001,Video,2026-09-16T18:02:08Z\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(1L, spool.counts()[PortabilityCategory.SUBSCRIPTIONS]) + assertEquals(2L, spool.counts()[PortabilityCategory.PLAYLISTS]) + val videos = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { videos += it } + assertEquals(1, videos.size) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + + private fun ZipOutputStream.entry(name: String, value: String) { + putNextEntry(ZipEntry(name)) + write(value.toByteArray()) + closeEntry() + } +} From ea9614e7e6b7209c4196a5cb9b31b288de233355 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 09:02:46 +0200 Subject: [PATCH 019/137] fix: report invalid Takeout dates --- .../YoutubeTakeoutHtmlPortabilityReader.kt | 14 +++++- .../YoutubeTakeoutJsonPortabilityReader.kt | 10 +++-- .../services/YoutubeTakeoutHistoryParser.kt | 20 +++++++-- .../services/YoutubeTakeoutParserService.kt | 7 +-- ...YoutubeTakeoutHtmlPortabilityReaderTest.kt | 44 +++++++++++++++++++ .../YoutubeTakeoutJsonSchemaDetectionTest.kt | 36 +++++++++++++++ 6 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt index 73636bbf..c36811df 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -27,8 +27,18 @@ internal object YoutubeTakeoutHtmlPortabilityReader { includeActivitySignals: Boolean, historyEntry: Boolean, ) { - YoutubeTakeoutHistoryParser.parse(html, requireWatchedMarker = !historyEntry) - .forEach { sink.write(it.toPortability()) } + val parsedHistory = YoutubeTakeoutHistoryParser.parseWithDiagnostics(html, requireWatchedMarker = !historyEntry) + parsedHistory.items.forEach { sink.write(it.toPortability()) } + if (parsedHistory.invalidDates > 0) { + sink.issue( + PortabilityIssue( + PortabilityCategory.HISTORY, + "invalid_takeout_date", + "YouTube Takeout HTML rows with an invalid date were skipped", + parsedHistory.invalidDates, + ), + ) + } if (includeActivitySignals) { val (subscriptions, favorites) = YoutubeTakeoutActivitySignalService.parseHtml(html) subscriptions.forEach { sink.write(it.toPortability()) } diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt index 5e729111..c338fbfc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt @@ -37,8 +37,7 @@ internal object YoutubeTakeoutJsonPortabilityReader { "myactivity json", "my activity json", ) || filename.startsWith("myactivity ") || filename.startsWith("my activity ") - return namedHistory || YoutubeTakeoutPathHints.isHistoryEntry(path) || - ("youtube" in normalized && ("activity" in normalized || "history" in normalized)) + return namedHistory || YoutubeTakeoutPathHints.isHistoryEntry(path) || "youtube" in normalized } fun read(input: InputStream, sink: PortabilityRecordSink) { @@ -48,7 +47,10 @@ internal object YoutubeTakeoutJsonPortabilityReader { } private fun read(parser: JsonParser, sink: PortabilityRecordSink) { - require(parser.nextToken() == JsonToken.START_ARRAY) { "YouTube Takeout JSON root must be an array" } + if (parser.nextToken() != JsonToken.START_ARRAY) { + parser.skipChildren() + return + } var count = 0 var invalidDates = 0L var historyRecords = 0L @@ -75,7 +77,7 @@ internal object YoutubeTakeoutJsonPortabilityReader { if (liked) sink.write(PortabilityFavorite(video, timestamp)) } else if (url != null && timestamp != null && liked) { sink.write(PortabilityFavorite(entry.video(url), timestamp)) - } else if (url != null && entry.time.isNotBlank() && history) { + } else if (url != null && history) { invalidDates += 1 } if (YoutubeTakeoutActivityClassifier.isSubscribedAction(entry.title) || diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index 0895a5e4..b7c70a85 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -12,9 +12,13 @@ object YoutubeTakeoutHistoryParser { private val tagRegex = Regex("<[^>]+>") private val spacesRegex = Regex("\\s+") - fun parse(html: String, requireWatchedMarker: Boolean = true): List { + fun parse(html: String, requireWatchedMarker: Boolean = true): List = + parseWithDiagnostics(html, requireWatchedMarker).items + + internal fun parseWithDiagnostics(html: String, requireWatchedMarker: Boolean = true): YoutubeTakeoutHistoryParseResult { val resolvedHtml = html.replace("\u00a0", " ") - return videoLinkRegex.findAll(resolvedHtml).mapNotNull { match -> + var invalidDates = 0L + val items = videoLinkRegex.findAll(resolvedHtml).mapNotNull { match -> val prefixStart = (match.range.first - WATCHED_PREFIX_CHARS).coerceAtLeast(0) if (requireWatchedMarker && !YoutubeTakeoutActivityClassifier.isWatched( resolvedHtml.substring(prefixStart, match.range.first), @@ -31,7 +35,11 @@ object YoutubeTakeoutHistoryParser { if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null val channelUrl = tail.groupValues[1].takeIf { it.isNotBlank() }.orEmpty() val channelName = decode(tail.groupValues[2]).ifBlank { "Unknown channel" } - val watchedAt = parseDate(decode(tail.groupValues[3])) ?: return@mapNotNull null + val dateText = decode(tail.groupValues[3]) + val watchedAt = parseDate(dateText) ?: run { + invalidDates += 1 + return@mapNotNull null + } HistoryItem( url = url, title = title, @@ -44,6 +52,7 @@ object YoutubeTakeoutHistoryParser { watchedAt = watchedAt, ) }.toList().distinctBy { it.url to it.watchedAt } + return YoutubeTakeoutHistoryParseResult(items, invalidDates) } private fun parseDate(value: String): Long? = YoutubeTakeoutDateParser.parseEpochMillis(value) @@ -68,3 +77,8 @@ object YoutubeTakeoutHistoryParser { private const val WATCHED_PREFIX_CHARS = 160 } + +internal data class YoutubeTakeoutHistoryParseResult( + val items: List, + val invalidDates: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt index 6b36ddb6..6de323d4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt @@ -99,12 +99,13 @@ class YoutubeTakeoutParserService { ?: entries.firstOrNull() if (entry == null) return emptyList() val html = zip.getInputStream(entry).bufferedReader().use { it.readText() } - val parsed = YoutubeTakeoutHistoryParser.parse( + val parsed = YoutubeTakeoutHistoryParser.parseWithDiagnostics( html, requireWatchedMarker = !YoutubeTakeoutPathHints.isHistoryEntry(entry.name), ) - if (parsed.isEmpty()) warnings += "No watch history rows detected" - return parsed.map(YoutubeTypeTypeMapper::historyItem) + if (parsed.items.isEmpty()) warnings += "No watch history rows detected" + if (parsed.invalidDates > 0) warnings += "Skipped ${parsed.invalidDates} watch history rows with invalid dates" + return parsed.items.map(YoutubeTypeTypeMapper::historyItem) } } diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt new file mode 100644 index 00000000..2e54d70e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutHtmlPortabilityReaderTest { + @TempDir + lateinit var directory: Path + + @Test + fun `reports invalid html dates without creating epoch history`() { + val archive = directory.resolve("takeout-invalid-date.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("Takeout/YouTube/Watch history.html")) + output.write( + ("You watched Video
" + + "16 Foo 2026, 12:00:00 CET
").toByteArray(), + ) + output.closeEntry() + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(0L, spool.counts()[PortabilityCategory.HISTORY] ?: 0L) + val issue = spool.issues().single() + assertEquals("invalid_takeout_date", issue.code) + assertEquals(1L, issue.count) + val history = mutableListOf() + spool.forEach(PortabilityCategory.HISTORY) { history += it } + assertTrue(history.isEmpty()) + } finally { + spool.delete() + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt new file mode 100644 index 00000000..0dbfccb6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonSchemaDetectionTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.portability + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Files +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class YoutubeTakeoutJsonSchemaDetectionTest { + @TempDir + lateinit var directory: Path + + @Test + fun `detects an activity array with an unknown localized filename`() { + val archive = directory.resolve("takeout-json.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.putNextEntry(ZipEntry("Takeout/YouTube/donnees.json")) + output.write( + """[{"title":"動画を視聴しました","titleUrl":"https://www.youtube.com/watch?v=unknown01","time":"2026-09-16T18:02:08Z"}]""".toByteArray(), + ) + output.closeEntry() + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + assertEquals(PortabilityFormat.YOUTUBE_TAKEOUT, YoutubeTakeoutPortabilityAdapter().detect(input)?.format) + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + assertEquals(1L, spool.counts()[PortabilityCategory.HISTORY]) + } finally { + spool.delete() + } + } +} From 302b76e5ba1ef5a248dc50d70bc143d72c053859 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 09:02:55 +0200 Subject: [PATCH 020/137] fix: publish Takeout import progress --- .../YoutubeTakeoutImportJobService.kt | 29 ++++++++++-- .../services/YoutubeTakeoutImportProgress.kt | 44 +++++++++++++++++++ .../services/YoutubeTakeoutImporterService.kt | 44 ++++++++++++++++--- .../YoutubeTakeoutSignalImportService.kt | 21 +++++++-- .../YoutubeTakeoutImportProgressTest.kt | 23 ++++++++++ 5 files changed, 150 insertions(+), 11 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt create mode 100644 src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt index e8a665b6..d94d39dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt @@ -5,6 +5,8 @@ import dev.typetype.server.models.YoutubeTakeoutImportReportItem import dev.typetype.server.models.YoutubeTakeoutPreviewItem import dev.typetype.server.models.YoutubeTakeoutCommitRequest import dev.typetype.server.models.YoutubeTakeoutCommitPlan +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import java.nio.file.Path @@ -60,7 +62,7 @@ class YoutubeTakeoutImportJobService( if (flags.importStarted) return statusStore.getStatus(userId, jobId) ?: error("Missing job") val plan = YoutubeTakeoutCommitPlanner.fromRequest(request) flagsStore.setImportStarted(jobId) - statusStore.updateStatus(jobId, "running", "importing", 75) + statusStore.updateStatus(jobId, "running", "importing", 0) engine.startCommit(jobId, plan) { runCommit(userId, jobId, it) } return statusStore.getStatus(userId, jobId) ?: error("Missing job") } @@ -69,7 +71,7 @@ class YoutubeTakeoutImportJobService( try { val flags = flagsStore.getFlags(userId, jobId) if (!flags.parseCompleted) preview(userId, jobId) - statusStore.updateStatus(jobId, "running", "importing", 75) + statusStore.updateStatus(jobId, "running", "importing", 0) val parsed = cache.getParsed(jobId) ?: parser.parse(Path.of(archiveStore.getArchivePath(userId, jobId))).also { cache.setParsed(jobId, it) val preview = previewService.build(userId, it) @@ -77,7 +79,19 @@ class YoutubeTakeoutImportJobService( cache.setPreview(jobId, preview) flagsStore.setParseCompleted(jobId) } - val report = importerService.commit(userId, parsed, plan) + val report = coroutineScope { + val progress = YoutubeTakeoutImportProgress(importTotal(parsed, plan)) + val publisher = launch { + progress.drain { value -> statusStore.updateStatus(jobId, "running", "importing", value) } + } + try { + importerService.commit(userId, parsed, plan) { processed, _ -> progress.offer(processed) } + .also { progress.finish() } + } finally { + progress.close() + publisher.join() + } + } reportStore.persistReport(jobId, Json.encodeToString(report)) flagsStore.setImportCompleted(jobId) statusStore.updateStatus(jobId, "completed", "completed", 100) @@ -95,6 +109,15 @@ class YoutubeTakeoutImportJobService( return "$name: $detail" } + private fun importTotal(parsed: dev.typetype.server.models.YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): Long = buildList { + if (plan.importSubscriptions) add(parsed.subscriptions.size.toLong()) + if (plan.importPlaylists) add(parsed.playlists.size.toLong()) + if (plan.importPlaylistItems) add(parsed.playlistItems.values.sumOf { it.size }.toLong()) + if (plan.importFavorites) add(parsed.favorites.size.toLong()) + if (plan.importWatchLater) add(parsed.watchLater.size.toLong()) + if (plan.importHistory) add(parsed.history.size.toLong()) + }.sum() + suspend fun report(userId: String, jobId: String): YoutubeTakeoutImportReportItem? { val reportJson = reportStore.getReport(userId, jobId) ?: return null return Json.decodeFromString(YoutubeTakeoutImportReportItem.serializer(), reportJson) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt new file mode 100644 index 00000000..94bf42cf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.channels.Channel +import java.util.concurrent.atomic.AtomicInteger + +internal class YoutubeTakeoutImportProgress(private val total: Long) { + private val updates = Channel(Channel.UNLIMITED) + private val published = AtomicInteger(-1) + + fun offer(processed: Long) { + val percent = if (total <= 0L) 99 else { + ((processed.toDouble() / total.toDouble()) * 99.0).toInt().coerceIn(0, 99) + } + publish(percent) + } + + fun finish() { + publish(99) + } + + fun close() { + updates.close() + } + + suspend fun drain(publish: suspend (Int) -> Unit) { + var applied = -1 + for (value in updates) { + if (value <= applied) continue + applied = value + publish(value) + } + } + + private fun publish(percent: Int) { + while (true) { + val current = published.get() + if (percent <= current) return + if (published.compareAndSet(current, percent)) { + updates.trySend(percent) + return + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt index e1beb779..179627af 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt @@ -9,6 +9,7 @@ import dev.typetype.server.models.YoutubeTakeoutImportStats import dev.typetype.server.models.YoutubeTakeoutParsedData import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import java.util.concurrent.atomic.AtomicLong class YoutubeTakeoutImporterService( private val subscriptionsService: SubscriptionsService, @@ -16,8 +17,16 @@ class YoutubeTakeoutImporterService( private val signalImportService: YoutubeTakeoutSignalImportService, private val playlistKeyService: YoutubeTakeoutPlaylistKeyService = YoutubeTakeoutPlaylistKeyService(), ) { - suspend fun commit(userId: String, parsed: YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): YoutubeTakeoutImportReportItem = coroutineScope { + suspend fun commit( + userId: String, + parsed: YoutubeTakeoutParsedData, + plan: YoutubeTakeoutCommitPlan, + onProgress: (processed: Long, total: Long) -> Unit = { _, _ -> }, + ): YoutubeTakeoutImportReportItem = coroutineScope { val (issues, issueSummary) = YoutubeTakeoutIssueService.build(parsed.warnings, parsed.errors, stage = "commit") + val total = importTotal(parsed, plan) + val processed = AtomicLong() + val progress = { onProgress(processed.incrementAndGet(), total) } val existingSubsDeferred = async { subscriptionsService.getAll(userId).map { it.channelUrl }.toSet() } val existingPlaylistsDeferred = async { playlistService.getAll(userId) } val sourceMappingsDeferred = async { playlistKeyService.getMappings(userId).toMutableMap() } @@ -35,6 +44,7 @@ class YoutubeTakeoutImporterService( existingSubs += canonicalUrl subImported += 1 } + progress() } } var plImported = 0 @@ -47,6 +57,7 @@ class YoutubeTakeoutImporterService( parsed.playlists.forEach { item -> if (YoutubeTakeoutSystemPlaylist.canonicalKey(item.name) != null || YoutubeTakeoutSystemPlaylist.canonicalKey(item.id) != null) { plSkipped += 1 + progress() return@forEach } val nameKey = item.name.lowercase() @@ -68,6 +79,7 @@ class YoutubeTakeoutImporterService( playlistKeyService.putMapping(userId, idKey, playlist.id) sourceMappings[idKey] = playlist.id } + progress() } } if (plan.importPlaylistItems) { @@ -75,11 +87,17 @@ class YoutubeTakeoutImporterService( val normalizedKey = playlistKey.lowercase() if (YoutubeTakeoutSystemPlaylist.canonicalKey(normalizedKey) != null) { itemSkipped += videos.size + repeat(videos.size) { progress() } return@forEach } val mappedId = sourceMappings[normalizedKey].orEmpty() val mappedPlaylist = if (mappedId.isBlank()) null else playlistService.getById(userId, mappedId) - val playlist = mappedPlaylist ?: createdBySource[normalizedKey] ?: return@forEach + val playlist = mappedPlaylist ?: createdBySource[normalizedKey] + if (playlist == null) { + itemSkipped += videos.size + repeat(videos.size) { progress() } + return@forEach + } val existingUrls = playlistService.getById(userId, playlist.id)?.videos?.map { it.url }.orEmpty().toMutableSet() videos.forEach { video -> if (video.url in existingUrls) itemSkipped += 1 else { @@ -87,12 +105,19 @@ class YoutubeTakeoutImporterService( itemImported += 1 existingUrls += video.url } + progress() } } } - val favoriteDeferred = if (plan.importFavorites) async { signalImportService.importFavorites(userId, favorites) } else null - val watchLaterDeferred = if (plan.importWatchLater) async { signalImportService.importWatchLater(userId, parsed.watchLater) } else null - val historyDeferred = if (plan.importHistory) async { signalImportService.importHistory(userId, parsed.history) } else null + val favoriteDeferred = if (plan.importFavorites) async { + signalImportService.importFavorites(userId, favorites) { progress() } + } else null + val watchLaterDeferred = if (plan.importWatchLater) async { + signalImportService.importWatchLater(userId, parsed.watchLater) { progress() } + } else null + val historyDeferred = if (plan.importHistory) async { + signalImportService.importHistory(userId, parsed.history) { progress() } + } else null val emptyStats = YoutubeTakeoutImportStats(0, 0, 0) val favoriteStats = favoriteDeferred?.await() ?: emptyStats val watchLaterStats = watchLaterDeferred?.await() ?: emptyStats @@ -114,4 +139,13 @@ class YoutubeTakeoutImporterService( if (plan.importSubscriptions) SubscriptionFeedCacheInvalidation.invalidate(userId) report } + + private fun importTotal(parsed: YoutubeTakeoutParsedData, plan: YoutubeTakeoutCommitPlan): Long = buildList { + if (plan.importSubscriptions) add(parsed.subscriptions.size.toLong()) + if (plan.importPlaylists) add(parsed.playlists.size.toLong()) + if (plan.importPlaylistItems) add(parsed.playlistItems.values.sumOf { it.size }.toLong()) + if (plan.importFavorites) add(parsed.favorites.size.toLong()) + if (plan.importWatchLater) add(parsed.watchLater.size.toLong()) + if (plan.importHistory) add(parsed.history.size.toLong()) + }.sum() } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt index 30412a7d..e9b09df4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt @@ -11,7 +11,11 @@ class YoutubeTakeoutSignalImportService( private val watchLaterService: WatchLaterService, private val historyService: HistoryService, ) { - suspend fun importFavorites(userId: String, items: List): YoutubeTakeoutImportStats { + suspend fun importFavorites( + userId: String, + items: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var imported = 0 var skipped = 0 val existing = favoritesService.getAll(userId).map { it.videoUrl }.toMutableSet() @@ -21,11 +25,16 @@ class YoutubeTakeoutSignalImportService( imported += 1 existing += item.videoUrl } + onProgress() } return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) } - suspend fun importWatchLater(userId: String, videos: List): YoutubeTakeoutImportStats { + suspend fun importWatchLater( + userId: String, + videos: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var imported = 0 var skipped = 0 val existing = watchLaterService.getAll(userId).map { it.url }.toMutableSet() @@ -35,11 +44,16 @@ class YoutubeTakeoutSignalImportService( imported += 1 existing += video.url } + onProgress() } return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) } - suspend fun importHistory(userId: String, items: List): YoutubeTakeoutImportStats { + suspend fun importHistory( + userId: String, + items: List, + onProgress: () -> Unit = {}, + ): YoutubeTakeoutImportStats { var skipped = 0 val existing = historyService.dedupKeys(userId).toMutableSet() val toInsert = mutableListOf() @@ -49,6 +63,7 @@ class YoutubeTakeoutSignalImportService( toInsert += item existing += key } + onProgress() } val imported = historyService.addImportedBatch(userId, toInsert) return YoutubeTakeoutImportStats(imported = imported, skipped = skipped, failed = 0) diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt new file mode 100644 index 00000000..7f8fef4a --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt @@ -0,0 +1,23 @@ +package dev.typetype.server + +import dev.typetype.server.services.YoutubeTakeoutImportProgress +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class YoutubeTakeoutImportProgressTest { + @Test + fun `publishes increasing percentages once and reaches ninety nine`() = runBlocking { + val progress = YoutubeTakeoutImportProgress(total = 10) + val observed = mutableListOf() + val draining = launch { progress.drain { observed += it } } + + repeat(10) { progress.offer(it + 1L) } + progress.finish() + progress.close() + draining.join() + + assertEquals((9..99 step 10).toList(), observed) + } +} From 426f4585773b3c6895b5b77259957724624eb4bc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 14:21:53 +0200 Subject: [PATCH 021/137] fix: skip channel tabs without tab support --- .../server/services/ChannelTabExtraction.kt | 5 ++++ .../server/services/PipePipeChannelService.kt | 4 +-- .../server/ChannelTabCapabilityTest.kt | 30 +++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt index 446fb83a..af6f20a2 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt +++ b/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt @@ -17,3 +17,8 @@ internal fun StreamingService.channelTabExtractor( val linkHandler = channelTabLHFactory.fromQuery(channelId, contentFilter, sort.toYouTubeChannelTabSortFilter()) return getChannelTabExtractor(linkHandler) } + +internal fun StreamingService.toChannelTab(url: String, sort: String?): String? { + if (channelTabLHFactory == null) return null + return url.toChannelTab(sort) +} diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt index 162d57ef..053939e6 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt @@ -61,7 +61,7 @@ class PipePipeChannelService : ChannelService { private fun extractFirstPage(url: String, sort: String?): ChannelResponse { val service = NewPipe.getServiceByUrl(url) - val tab = url.toChannelTab(sort) + val tab = service.toChannelTab(url, sort) if (tab != null) { val channelUrl = url.toBaseChannelUrl(tab) val metadata = runCatching { ChannelInfo.getInfo(channelUrl) }.getOrNull() @@ -74,7 +74,7 @@ class PipePipeChannelService : ChannelService { private fun extractMorePage(url: String, page: Page, sort: String?): ChannelResponse { val service = NewPipe.getServiceByUrl(url) - val tab = url.toChannelTab(sort) + val tab = service.toChannelTab(url, sort) if (tab != null) { val extractor = service.channelTabExtractor(url, channelId(url.toBaseChannelUrl(tab), service), tab, sort) return extractor.getPage(page).toChannelTabResponse() diff --git a/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt b/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt new file mode 100644 index 00000000..fbdc0031 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ChannelTabCapabilityTest.kt @@ -0,0 +1,30 @@ +package dev.typetype.server + +import dev.typetype.server.services.toChannelTab +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.linkhandler.ChannelTabs + +class ChannelTabCapabilityTest { + @Test + fun `services without channel tab support fall back to channel info`() { + val tab = ServiceList.BiliBili.toChannelTab( + "https://space.bilibili.com/946974", + "latest", + ) + + assertNull(tab) + } + + @Test + fun `services with channel tab support keep sorted videos`() { + val tab = ServiceList.YouTube.toChannelTab( + "https://www.youtube.com/@test", + "latest", + ) + + assertEquals(ChannelTabs.VIDEOS, tab) + } +} From e330e65696ccb2cce5347deaa05575346ab3591c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 16:18:17 +0200 Subject: [PATCH 022/137] fix: associate localized playlist item files --- .../YoutubeTakeoutCsvPortabilityReader.kt | 11 +++++ .../YoutubeTakeoutPortabilityAdapterTest.kt | 40 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index f514c62d..7b2e7249 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -113,6 +113,17 @@ internal object YoutubeTakeoutCsvPortabilityReader { private fun playlistAliases(name: String): Set = setOf( name, + ) + filenameAliases(name) + localizedPlaylistAliases(name) + + private fun filenameAliases(name: String): Set { + val escaped = name.replace('/', '_').replace('\'', '_') + return setOf( + "$escaped-videos", + "$escaped-vídeos", + ) + } + + private fun localizedPlaylistAliases(name: String): Set = setOf( "Videos from $name", "Videos de $name", "Vídeos de $name", diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt index f02ac77a..af6624ce 100644 --- a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt +++ b/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt @@ -103,6 +103,46 @@ class YoutubeTakeoutPortabilityAdapterTest { spool.delete() } + @Test + fun `adapter associates spanish playlist item files with manifest ids`() { + val archive = directory.resolve("takeout-es-manifest.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Listas de reproducción.csv", + "ID de lista de reproducción,Título de la lista de reproducción (original)\n" + + "PL123456789,Importada\nPL987654321,It's/Luna\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/Importada-vídeos.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\n" + + "video000001,2026-01-02T00:00:00Z\n" + + "video000002,2026-01-01T00:00:00Z\n", + ) + output.entry( + "Takeout/YouTube y YouTube Music/listas de reproducción/It_s_Luna-vídeos.csv", + "ID de vídeo,Marca de tiempo de creación de la lista de reproducción\nvideo000003,2026-01-03T00:00:00Z\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(5L, spool.counts()[PortabilityCategory.PLAYLISTS]) + val videos = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL123456789") { record -> + videos += record as PortabilityPlaylistVideo + } + assertEquals(listOf("video000001", "video000002"), videos.map { it.video.url.substringAfter("v=") }) + val escaped = mutableListOf() + spool.forEachChild(PortabilityCategory.PLAYLISTS, "PL987654321") { record -> + escaped += record as PortabilityPlaylistVideo + } + assertEquals(listOf("video000003"), escaped.map { it.video.url.substringAfter("v=") }) + assertTrue(spool.issues().none { it.code == "playlist_manifest_missing" }) + spool.delete() + } + @Test fun `adapter streams My Activity JSON including embedded URLs`() { val json = directory.resolve("watch-history.json") From 5e2c138d4aacbf0a327ed8b241df6d5ceaa8ac8b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 16:18:17 +0200 Subject: [PATCH 023/137] fix: parse one-digit takeout hours --- .../dev/typetype/server/services/YoutubeTakeoutDateParser.kt | 4 ++++ .../dev/typetype/server/YoutubeTakeoutParserServiceTest.kt | 1 + 2 files changed, 5 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt index 25953258..4b6d0395 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt @@ -14,9 +14,13 @@ import java.util.Locale object YoutubeTakeoutDateParser { private val ACTIVITY_PATTERNS = listOf( "d MMM yyyy, HH:mm:ss z", + "d MMM yyyy, H:mm:ss z", "d MMMM yyyy, HH:mm:ss z", + "d MMMM yyyy, H:mm:ss z", "d. MMM yyyy, HH:mm:ss z", + "d. MMM yyyy, H:mm:ss z", "d. MMMM yyyy, HH:mm:ss z", + "d. MMMM yyyy, H:mm:ss z", "MMM d, yyyy, HH:mm:ss z", "MMMM d, yyyy, HH:mm:ss z", "d MMM yyyy, h:mm:ss a z", diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt index 0fcb97db..5c3fcb04 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt @@ -102,6 +102,7 @@ class YoutubeTakeoutParserServiceTest { assertEquals(1_789_574_528_000L, parsed.history.single().watchedAt) assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 septiembre 2026, 18:02:08 CEST")) assertEquals(1_789_574_528_000L, YoutubeTakeoutDateParser.parseEpochMillis("16 de septiembre de 2026, 18:02:08 CEST")) + assertEquals(1_789_250_639_000L, YoutubeTakeoutDateParser.parseEpochMillis("13 sept 2026, 0:03:59 CEST")) Files.deleteIfExists(zip) } From 640540ee938ff2c08adaa7a00082650887481f5b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 16:18:17 +0200 Subject: [PATCH 024/137] perf: anchor takeout history row matching --- .../typetype/server/services/YoutubeTakeoutHistoryParser.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index b7c70a85..51d06973 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -27,9 +27,7 @@ object YoutubeTakeoutHistoryParser { return@mapNotNull null } val tailStart = match.range.last + 1 - val tail = rowTailRegex.find(resolvedHtml, tailStart) - ?.takeIf { it.range.first == tailStart } - ?: return@mapNotNull null + val tail = rowTailRegex.matchAt(resolvedHtml, tailStart) ?: return@mapNotNull null val url = extractUrl(match.groupValues[1]) ?: return@mapNotNull null val title = decode(match.groupValues[2]) if (YoutubeTakeoutUnavailableItem.matches(title)) return@mapNotNull null From 74a07b2fd1eec3a9e626d2abd8f06beaf44d402a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 17 Sep 2026 17:43:16 +0200 Subject: [PATCH 025/137] feat: separate autoplay open setting --- openapi/components/access-control.yaml | 1 + .../server/db/SettingsSchemaMigrations.kt | 2 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../services/SettingsPersistenceMappers.kt | 2 + .../SettingsAutoplayOnOpenRoutesTest.kt | 72 +++++++++++++++++++ 6 files changed, 79 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 5e3d7d25..9c0d05bc 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -11,6 +11,7 @@ SettingsItem: default: 1 defaultLandingPage: { type: string, default: home } autoplay: { type: boolean, default: true } + autoplayOnOpen: { type: boolean, default: true } volume: { type: number, format: double, default: 1.0 } muted: { type: boolean, default: false } notificationPopupsEnabled: { type: boolean, default: true } diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index e49baf63..b6152524 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -29,6 +29,8 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_members_only_content BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_on_open BOOLEAN NOT NULL DEFAULT true") + exec("UPDATE settings SET autoplay_on_open = autoplay") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_landing_page TEXT NOT NULL DEFAULT 'home'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 253b348f..f540816c 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -9,6 +9,7 @@ object SettingsTable : Table("settings") { val defaultPlaybackSpeed = double("default_playback_speed").default(1.0) val defaultLandingPage = text("default_landing_page").default("home") val autoplay = bool("autoplay").default(true) + val autoplayOnOpen = bool("autoplay_on_open").default(true) val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) val volume = double("volume").default(1.0) val muted = bool("muted").default(false) diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 5d5d7447..04eb8e49 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -9,6 +9,7 @@ data class SettingsItem( val defaultPlaybackSpeed: Double = 1.0, val defaultLandingPage: String = "home", val autoplay: Boolean = true, + val autoplayOnOpen: Boolean = true, val skipPlaylistAutoplayScreen: Boolean = false, val volume: Double = 1.0, val muted: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index e2087004..8963c806 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -22,6 +22,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultPlaybackSpeed = this[SettingsTable.defaultPlaybackSpeed], defaultLandingPage = this[SettingsTable.defaultLandingPage], autoplay = this[SettingsTable.autoplay], + autoplayOnOpen = this[SettingsTable.autoplayOnOpen], skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], volume = this[SettingsTable.volume], muted = this[SettingsTable.muted], @@ -62,6 +63,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultPlaybackSpeed] = settings.defaultPlaybackSpeed this[SettingsTable.defaultLandingPage] = settings.defaultLandingPage this[SettingsTable.autoplay] = settings.autoplay + this[SettingsTable.autoplayOnOpen] = settings.autoplayOnOpen this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen this[SettingsTable.volume] = settings.volume this[SettingsTable.muted] = settings.muted diff --git a/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt new file mode 100644 index 00000000..9215d413 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt @@ -0,0 +1,72 @@ +package dev.typetype.server + +import dev.typetype.server.routes.settingsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SettingsService +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SettingsAutoplayOnOpenRoutesTest { + private val service = SettingsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() { TestDatabase.setup() } + } + + @BeforeEach + fun clean() { TestDatabase.truncateAll() } + + @Test + fun `autoplay on open defaults independently from autoplay next`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + assertEquals(HttpStatusCode.OK, response.status) + val body = response.bodyAsText() + assertTrue(body.contains("\"autoplay\":true")) + assertTrue(body.contains("\"autoplayOnOpen\":true")) + } + + @Test + fun `autoplay next can be disabled while autoplay on open stays enabled`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"autoplay":false,"autoplayOnOpen":true}""") + } + val body = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText() + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"autoplay\":false")) + assertTrue(body.contains("\"autoplayOnOpen\":true")) + } +} From 2680d9a6ed68d8255ab1f426767c8b9dad1d3c2a Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 10:45:39 +0200 Subject: [PATCH 026/137] docs: link AI transparency statement --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0886b3b9..72617033 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +U want to know the current position of TypeType about AI ? Go check [this](https://github.com/TypeType-Video/TypeType/blob/dev/AI_TRANSPARENCY.md). +
TypeType

TypeType Server

From d74fb35a931e432dea448020df8f90e61d1d9f52 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 10:57:13 +0200 Subject: [PATCH 027/137] docs: move AI transparency notice --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72617033..f3c9420e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -U want to know the current position of TypeType about AI ? Go check [this](https://github.com/TypeType-Video/TypeType/blob/dev/AI_TRANSPARENCY.md). -
TypeType

TypeType Server

Extraction, API, and private user data backend for TypeType.

+You want to know the current position of TypeType about AI ? Go check [this](https://github.com/TypeType-Video/TypeType/blob/dev/AI_TRANSPARENCY.md). + TypeType-Server is the Kotlin/Ktor HTTP API behind TypeType. It wraps [PipePipeExtractor](https://github.com/InfinityLoop1308/PipePipeExtractor) for supported media services and stores instance and user data in PostgreSQL. If you want to run a complete TypeType instance, use the [central stack](https://github.com/TypeType-Video/TypeType) rather than deploying this service alone. From 4ad2488d571192ac7586e16795f8311470e60024 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:20:55 +0200 Subject: [PATCH 028/137] refactor: extract server-cache Gradle module --- build.gradle.kts | 2 +- server-cache/build.gradle.kts | 14 ++++++++++++++ .../kotlin/dev/typetype/server/cache/CacheJson.kt | 2 +- .../dev/typetype/server/cache/CacheService.kt | 0 .../dev/typetype/server/cache/DragonflyService.kt | 0 settings.gradle.kts | 10 ++++++++++ 6 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 server-cache/build.gradle.kts rename {src => server-cache/src}/main/kotlin/dev/typetype/server/cache/CacheJson.kt (53%) rename {src => server-cache/src}/main/kotlin/dev/typetype/server/cache/CacheService.kt (100%) rename {src => server-cache/src}/main/kotlin/dev/typetype/server/cache/DragonflyService.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index fe90f2be..18038483 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,6 +25,7 @@ repositories { } dependencies { + implementation(project(":server-cache")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) @@ -48,7 +49,6 @@ dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") - implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("com.zaxxer:HikariCP:7.1.0") diff --git a/server-cache/build.gradle.kts b/server-cache/build.gradle.kts new file mode 100644 index 00000000..8c743aa1 --- /dev/null +++ b/server-cache/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api("io.lettuce:lettuce-core:7.7.0.RELEASE") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt similarity index 53% rename from src/main/kotlin/dev/typetype/server/cache/CacheJson.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt index 4af329b2..b864ae08 100644 --- a/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt +++ b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheJson.kt @@ -2,4 +2,4 @@ package dev.typetype.server.cache import kotlinx.serialization.json.Json -internal val CacheJson: Json = Json { ignoreUnknownKeys = true } +val CacheJson: Json = Json { ignoreUnknownKeys = true } diff --git a/src/main/kotlin/dev/typetype/server/cache/CacheService.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/CacheService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/cache/CacheService.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/CacheService.kt diff --git a/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt b/server-cache/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt rename to server-cache/src/main/kotlin/dev/typetype/server/cache/DragonflyService.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 0a997ae6..d2d522be 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,5 +1,15 @@ rootProject.name = "typetype-server" +dependencyResolutionManagement { + repositories { + mavenLocal() + mavenCentral() + maven { url = uri("https://jitpack.io") } + } +} + +include(":server-cache") + val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull ?.let { file(it) } From e6b048ef4e13957cf2f29356c23f809eed18e4f2 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:25:36 +0200 Subject: [PATCH 029/137] refactor: extract server-core models into Gradle module --- build.gradle.kts | 1 + server-core/build.gradle.kts | 12 ++++++++++++ .../typetype/server/models/AccountIdentityModels.kt | 0 .../dev/typetype/server/models/AccountProfileItem.kt | 0 .../server/models/AccountProfilesResponse.kt | 0 .../dev/typetype/server/models/ActiveSessionItem.kt | 0 .../server/models/ActiveSessionNowPlayingItem.kt | 0 .../typetype/server/models/AdminAllowListUserItem.kt | 0 .../server/models/AdminManagedAccessUserItem.kt | 0 .../server/models/AdminManagedAccessUsersResponse.kt | 0 .../typetype/server/models/AdminUserAllowListItem.kt | 0 .../dev/typetype/server/models/AdminUserItem.kt | 0 .../dev/typetype/server/models/AdminUsersPageItem.kt | 0 .../dev/typetype/server/models/AllowedChannelItem.kt | 0 .../typetype/server/models/AllowedPlaylistItem.kt | 0 .../server/models/AudioOnlyStreamResponse.kt | 0 .../dev/typetype/server/models/AudioStreamItem.kt | 0 .../typetype/server/models/AvatarCustomRequest.kt | 0 .../dev/typetype/server/models/AvatarEmojiRequest.kt | 0 .../kotlin/dev/typetype/server/models/BlockedItem.kt | 0 .../dev/typetype/server/models/BlockedKeywordItem.kt | 0 .../dev/typetype/server/models/BugReportContext.kt | 0 .../dev/typetype/server/models/BugReportRequest.kt | 0 .../dev/typetype/server/models/BugReportResponse.kt | 0 .../dev/typetype/server/models/BulletCommentItem.kt | 0 .../server/models/BulletCommentsPageResponse.kt | 0 .../dev/typetype/server/models/CaptionStylesItem.kt | 0 .../dev/typetype/server/models/ChannelPageRequest.kt | 0 .../server/models/ChannelPlaylistsResponse.kt | 0 .../dev/typetype/server/models/ChannelResponse.kt | 0 .../dev/typetype/server/models/ChannelResultItem.kt | 0 .../kotlin/dev/typetype/server/models/CommentItem.kt | 0 .../typetype/server/models/CommentsPageResponse.kt | 0 .../dev/typetype/server/models/CustomAvatarItem.kt | 0 .../kotlin/dev/typetype/server/models/DeArrowItem.kt | 0 .../server/models/DeArrowThumbnailCandidate.kt | 0 .../typetype/server/models/DeArrowTitleCandidate.kt | 0 .../dev/typetype/server/models/DeepHealthResponse.kt | 0 .../dev/typetype/server/models/ExtractionResult.kt | 0 .../dev/typetype/server/models/FavoriteItem.kt | 0 .../dev/typetype/server/models/HealthResponse.kt | 0 .../kotlin/dev/typetype/server/models/HistoryItem.kt | 0 .../server/models/HomeRecommendationsDebug.kt | 0 .../server/models/HomeRecommendationsResponse.kt | 0 .../server/models/InstanceMinClientVersion.kt | 0 .../dev/typetype/server/models/InstanceResponse.kt | 0 .../server/models/MarkNotificationsReadResponse.kt | 0 .../dev/typetype/server/models/NotificationItem.kt | 0 .../typetype/server/models/NotificationsResponse.kt | 0 .../typetype/server/models/OidcCallbackRequest.kt | 0 .../typetype/server/models/OidcCallbackResponse.kt | 0 .../dev/typetype/server/models/OidcPublicConfig.kt | 0 .../dev/typetype/server/models/OidcStartResponse.kt | 0 .../dev/typetype/server/models/OidcStatusResponse.kt | 0 .../dev/typetype/server/models/PlaylistItem.kt | 0 .../typetype/server/models/PlaylistReorderRequest.kt | 0 .../typetype/server/models/PlaylistReorderResult.kt | 0 .../dev/typetype/server/models/PlaylistResultItem.kt | 0 .../dev/typetype/server/models/PlaylistVideoItem.kt | 0 .../server/models/PodcastEpisodesResponse.kt | 0 .../kotlin/dev/typetype/server/models/PodcastItem.kt | 0 .../typetype/server/models/PodcastPageResponse.kt | 0 .../dev/typetype/server/models/PresenceModels.kt | 0 .../dev/typetype/server/models/PreviewFrameItem.kt | 0 .../dev/typetype/server/models/ProfileNameRequest.kt | 0 .../typetype/server/models/ProfileSwitchResponse.kt | 0 .../typetype/server/models/ProfileUpdateRequest.kt | 0 .../dev/typetype/server/models/ProgressItem.kt | 0 .../dev/typetype/server/models/PublicPlaylistItem.kt | 0 .../typetype/server/models/PublicPlaylistResponse.kt | 0 .../dev/typetype/server/models/PublicProfileItem.kt | 0 .../typetype/server/models/PushNotificationModels.kt | 0 .../server/models/RecommendationEventItem.kt | 0 .../server/models/RecommendationEventRequest.kt | 0 .../server/models/RecommendationFeedbackItem.kt | 0 .../server/models/RecommendationFeedbackRequest.kt | 0 .../server/models/RecommendationOnboardingModels.kt | 0 .../server/models/RestorePipePipeResultItem.kt | 0 .../dev/typetype/server/models/RssFeedModels.kt | 0 .../dev/typetype/server/models/SavedPlaylistItem.kt | 0 .../typetype/server/models/SavedPlaylistRequest.kt | 0 .../dev/typetype/server/models/SearchFilterGroup.kt | 0 .../dev/typetype/server/models/SearchFilterOption.kt | 0 .../typetype/server/models/SearchFiltersResponse.kt | 0 .../dev/typetype/server/models/SearchHistoryItem.kt | 0 .../dev/typetype/server/models/SearchPageResponse.kt | 0 .../typetype/server/models/SessionActivityRequest.kt | 0 .../server/models/SessionPlaybackProgressRequest.kt | 0 .../server/models/SessionPlaybackStartRequest.kt | 0 .../server/models/SessionPlaybackStopRequest.kt | 0 .../dev/typetype/server/models/SettingsItem.kt | 0 .../server/models/SponsorBlockCategoryActions.kt | 0 .../dev/typetype/server/models/SponsorBlockMode.kt | 0 .../server/models/SponsorBlockSegmentItem.kt | 0 .../dev/typetype/server/models/StreamResponse.kt | 0 .../dev/typetype/server/models/StreamSegmentItem.kt | 0 .../server/models/SubscriptionCreateRequest.kt | 0 .../server/models/SubscriptionFeedResponse.kt | 0 .../server/models/SubscriptionGroupBackupItem.kt | 0 .../typetype/server/models/SubscriptionGroupItem.kt | 0 .../server/models/SubscriptionGroupMembershipItem.kt | 0 .../models/SubscriptionGroupMembershipRequest.kt | 0 .../server/models/SubscriptionGroupRequest.kt | 0 .../dev/typetype/server/models/SubscriptionItem.kt | 0 .../dev/typetype/server/models/SubtitleItem.kt | 0 .../dev/typetype/server/models/TypeTypeBackupItem.kt | 0 .../typetype/server/models/UnreadCountResponse.kt | 0 .../dev/typetype/server/models/UserProfileItem.kt | 0 .../dev/typetype/server/models/VersionResponse.kt | 0 .../kotlin/dev/typetype/server/models/VideoItem.kt | 0 .../dev/typetype/server/models/VideoStreamItem.kt | 0 .../dev/typetype/server/models/WatchLaterItem.kt | 0 .../models/YoutubeRemoteBrowserCompleteRequest.kt | 0 .../models/YoutubeRemoteBrowserStartRequest.kt | 0 .../models/YoutubeRemoteBrowserStartResponse.kt | 0 .../server/models/YoutubeRemoteLoginStatus.kt | 0 .../server/models/YoutubeSessionCompleteRequest.kt | 0 .../server/models/YoutubeSessionPairingResponse.kt | 0 .../server/models/YoutubeSessionStatusResponse.kt | 0 .../server/models/YoutubeTakeoutCommitPlan.kt | 0 .../server/models/YoutubeTakeoutCommitRequest.kt | 0 .../typetype/server/models/YoutubeTakeoutCsvRows.kt | 0 .../server/models/YoutubeTakeoutImportJobStatus.kt | 0 .../server/models/YoutubeTakeoutImportReportItem.kt | 0 .../server/models/YoutubeTakeoutIssueItem.kt | 0 .../server/models/YoutubeTakeoutParsedData.kt | 0 .../server/models/YoutubeTakeoutPreviewItem.kt | 0 .../server/models/YoutubeTakeoutPreviewSamples.kt | 0 settings.gradle.kts | 1 + .../server/routes/RecommendationEventsRoutes.kt | 9 ++++++--- .../typetype/server/services/BugReportValidation.kt | 6 ++++-- .../typetype/server/services/GitHubIssueService.kt | 3 ++- .../typetype/server/services/RssFeedReaderService.kt | 3 ++- .../server/services/SubscriptionFeedOrderer.kt | 3 +-- 134 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 server-core/build.gradle.kts rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminUserItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BlockedItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BugReportContext.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BugReportRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BugReportResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ChannelResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/CommentItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/DeArrowItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ExtractionResult.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/FavoriteItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/HealthResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/HistoryItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/InstanceResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/NotificationItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PlaylistItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PodcastItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PresenceModels.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ProgressItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/RssFeedModels.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SettingsItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/StreamResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/SubtitleItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/UserProfileItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/VersionResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/VideoItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 18038483..6cc6d482 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,6 +26,7 @@ repositories { dependencies { implementation(project(":server-cache")) + implementation(project(":server-core")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) diff --git a/server-core/build.gradle.kts b/server-core/build.gradle.kts new file mode 100644 index 00000000..c0035dab --- /dev/null +++ b/server-core/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountIdentityModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AccountProfilesResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ActiveSessionNowPlayingItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminAllowListUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminManagedAccessUsersResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUserAllowListItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUserItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminUsersPageItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AllowedChannelItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AllowedPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AudioOnlyStreamResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AudioStreamItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AvatarCustomRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AvatarEmojiRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BlockedItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BlockedItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BlockedKeywordItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportContext.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportContext.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BugReportResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/BulletCommentsPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CaptionStylesItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelPageRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelPlaylistsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ChannelResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CommentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CommentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CommentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CommentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CommentsPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/CustomAvatarItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowThumbnailCandidate.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeArrowTitleCandidate.kt diff --git a/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/DeepHealthResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt diff --git a/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/FavoriteItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HealthResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HealthResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HistoryItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HistoryItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsDebug.kt diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/HomeRecommendationsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt b/server-core/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/InstanceMinClientVersion.kt diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/MarkNotificationsReadResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/NotificationItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/NotificationItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/NotificationsResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcCallbackResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcPublicConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/OidcStatusResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistReorderResult.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PlaylistVideoItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastEpisodesResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PodcastPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PresenceModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PresenceModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PreviewFrameItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileNameRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileSwitchResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProfileUpdateRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProgressItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProgressItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicPlaylistResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PublicProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/PushNotificationModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationEventRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationFeedbackRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RecommendationOnboardingModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RestorePipePipeResultItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt b/server-core/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SavedPlaylistRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterGroup.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFilterOption.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchFiltersResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchHistoryItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SearchPageResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionActivityRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackProgressRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SessionPlaybackStopRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SettingsItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockCategoryActions.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockMode.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SponsorBlockSegmentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/StreamResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/StreamResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/StreamSegmentItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionCreateRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionFeedResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupBackupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/SubtitleItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/TypeTypeBackupItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/UnreadCountResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/UserProfileItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VersionResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VersionResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VideoItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VideoItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VideoItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VideoItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/VideoStreamItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/WatchLaterItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserCompleteRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteBrowserStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeRemoteLoginStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionCompleteRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionPairingResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeSessionStatusResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitPlan.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCommitRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutCsvRows.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportJobStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutImportReportItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutIssueItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutParsedData.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt b/server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/YoutubeTakeoutPreviewSamples.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index d2d522be..f5aa07a2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -9,6 +9,7 @@ dependencyResolutionManagement { } include(":server-cache") +include(":server-core") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt index 14abe962..ff8bbc89 100644 --- a/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt @@ -31,13 +31,16 @@ fun Route.recommendationEventsRoutes(eventService: RecommendationEventService, a if ((request.eventType == "click" || request.eventType == "watch") && request.videoUrl.isNullOrBlank()) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoUrl")) } - if (request.watchRatio != null && (request.watchRatio < 0.0 || request.watchRatio > 1.0)) { + val watchRatio = request.watchRatio + if (watchRatio != null && (watchRatio < 0.0 || watchRatio > 1.0)) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid watchRatio")) } - if (request.watchDurationMs != null && request.watchDurationMs < 0) { + val watchDurationMs = request.watchDurationMs + if (watchDurationMs != null && watchDurationMs < 0) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid watchDurationMs")) } - if (request.contextKey != null && request.contextKey.length > 120) { + val contextKeyValue = request.contextKey + if (contextKeyValue != null && contextKeyValue.length > 120) { return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid contextKey")) } val contextKey = request.contextKey ?: HomeRecommendationContextualBandit.contextKey( diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt index e19625ff..e1c6d485 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt +++ b/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt @@ -36,10 +36,12 @@ internal object BugReportValidation { context.screenHeight, ).filterNotNull() if (dimensions.any { it !in 1..100_000 }) return "Invalid display dimensions" - if (context.devicePixelRatio != null && context.devicePixelRatio !in 0.1..100.0) { + val devicePixelRatio = context.devicePixelRatio + if (devicePixelRatio != null && devicePixelRatio !in 0.1..100.0) { return "Invalid device pixel ratio" } - if (context.timezone != null && context.timezone.length > 128) return "Invalid timezone" + val timezone = context.timezone + if (timezone != null && timezone.length > 128) return "Invalid timezone" return null } } diff --git a/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt b/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt index dd06bcc9..140ab4d0 100644 --- a/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt +++ b/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt @@ -54,7 +54,8 @@ class GitHubIssueService( appendLine("## API errors") report.context.apiErrors.take(10).forEach { error -> appendLine("- endpoint=${redactDomains(error.endpoint)} status=${error.status} requestId=${redactDomains(error.requestId ?: "n/a")} code=${error.code ?: "n/a"} timestamp=${error.timestamp}") - if (!error.message.isNullOrBlank()) appendLine(" message=${redactDomains(error.message)}") + val errorMessage = error.message + if (!errorMessage.isNullOrBlank()) appendLine(" message=${redactDomains(errorMessage)}") } } } diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt index eae437df..18fc0b95 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt @@ -36,7 +36,8 @@ class RssFeedReaderService internal constructor( val lastModified = RssDocumentRenderer.lastModified(stored.item, videos, now) val bytes = RssDocumentRenderer.render(stored.item, videos, baseUrl, lastModified) val etag = "\"${HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes))}\"" - if (stored.item.lastUsedAt == null || now - stored.item.lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { + val lastUsedAt = stored.item.lastUsedAt + if (lastUsedAt == null || now - lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { repository.touch(feedId, now) } return RssFeedReadResult.Ready(bytes, etag, lastModified, config.rssMinimumPollMinutes * 60) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt index a1dc2b9d..89f51501 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -41,8 +41,7 @@ internal class SubscriptionFeedOrderer { private fun VideoItem.feedTimestamp(): Long = when { uploaded >= 0L -> uploaded - publishedAt != null && publishedAt >= 0L -> publishedAt - else -> Long.MIN_VALUE + else -> publishedAt?.takeIf { it >= 0L } ?: Long.MIN_VALUE } } From 8e3a033a4931d7ef6e9fd4249a012f6025ef2ccf Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:27:21 +0200 Subject: [PATCH 030/137] refactor: extract server-db into Gradle module --- build.gradle.kts | 1 + .../dev/typetype/server/InstanceDefaults.kt | 0 .../server/services/ChannelUrlCanonicalizer.kt | 0 server-db/build.gradle.kts | 16 ++++++++++++++++ .../db/DatabaseCollectionMetadataMigration.kt | 0 .../dev/typetype/server/db/DatabaseFactory.kt | 0 .../db/DatabaseImportedMediaRepairMigration.kt | 0 .../server/db/DatabaseIndexMigrations.kt | 0 .../typetype/server/db/DatabaseOidcMigration.kt | 0 .../server/db/DatabasePrimaryKeyMigrations.kt | 0 .../db/DatabaseProfileAccountsMigration.kt | 0 .../server/db/DatabaseSessionAuthMigration.kt | 0 .../DatabaseSubscriptionsCanonicalMigration.kt | 0 .../db/DatabaseYoutubeRemoteLoginMigration.kt | 0 .../server/db/SettingsSchemaMigrations.kt | 0 .../server/db/tables/AdminSettingsTable.kt | 0 .../server/db/tables/AllowedChannelsTable.kt | 0 .../server/db/tables/AllowedPlaylistsTable.kt | 0 .../server/db/tables/BlockedChannelsTable.kt | 0 .../server/db/tables/BlockedKeywordsTable.kt | 0 .../server/db/tables/BlockedVideosTable.kt | 0 .../typetype/server/db/tables/BugReportsTable.kt | 0 .../ChannelNotificationPreferencesTable.kt | 0 .../typetype/server/db/tables/FavoritesTable.kt | 0 .../typetype/server/db/tables/HistoryTable.kt | 0 .../db/tables/NotificationReadItemsTable.kt | 0 .../server/db/tables/NotificationStatesTable.kt | 0 .../server/db/tables/PasswordResetTable.kt | 0 .../server/db/tables/PlaylistVideosTable.kt | 0 .../typetype/server/db/tables/PlaylistsTable.kt | 0 .../server/db/tables/PresenceKeysTable.kt | 0 .../server/db/tables/ProfileAccountsTable.kt | 0 .../typetype/server/db/tables/ProgressTable.kt | 0 .../server/db/tables/PushDevicesTable.kt | 0 .../db/tables/PushNotificationBaselinesTable.kt | 0 .../db/tables/PushNotificationDeliveriesTable.kt | 0 .../db/tables/PushNotificationEventsTable.kt | 0 .../db/tables/PushNotificationSeenVideosTable.kt | 0 .../db/tables/RecommendationEventsTable.kt | 0 .../db/tables/RecommendationFeedHistoryTable.kt | 0 .../db/tables/RecommendationFeedbackTable.kt | 0 .../RecommendationOnboardingPreferencesTable.kt | 0 .../tables/RecommendationOnboardingStateTable.kt | 0 .../server/db/tables/RssFeedChannelsTable.kt | 0 .../server/db/tables/RssFeedServicesTable.kt | 0 .../typetype/server/db/tables/RssFeedsTable.kt | 0 .../server/db/tables/RssUserPoliciesTable.kt | 0 .../server/db/tables/SavedPlaylistsTable.kt | 0 .../server/db/tables/SearchHistoryTable.kt | 0 .../typetype/server/db/tables/SessionsTable.kt | 0 .../typetype/server/db/tables/SettingsTable.kt | 0 .../tables/SubscriptionGroupMembershipsTable.kt | 0 .../server/db/tables/SubscriptionGroupsTable.kt | 0 .../server/db/tables/SubscriptionsTable.kt | 0 .../server/db/tables/UserAvatarsTable.kt | 0 .../server/db/tables/UserChannelInterestTable.kt | 0 .../server/db/tables/UserTopicInterestTable.kt | 0 .../dev/typetype/server/db/tables/UsersTable.kt | 0 .../typetype/server/db/tables/WatchLaterTable.kt | 0 .../db/tables/YoutubeSessionPairingsTable.kt | 0 .../server/db/tables/YoutubeSessionsTable.kt | 0 .../db/tables/YoutubeTakeoutImportJobsTable.kt | 0 .../db/tables/YoutubeTakeoutPlaylistKeysTable.kt | 0 settings.gradle.kts | 1 + 64 files changed, 18 insertions(+) rename {src => server-core/src}/main/kotlin/dev/typetype/server/InstanceDefaults.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt (100%) create mode 100644 server-db/build.gradle.kts rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt (100%) rename {src => server-db/src}/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 6cc6d482..03df8c71 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -27,6 +27,7 @@ repositories { dependencies { implementation(project(":server-cache")) implementation(project(":server-core")) + implementation(project(":server-db")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) diff --git a/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt b/server-core/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/InstanceDefaults.kt rename to server-core/src/main/kotlin/dev/typetype/server/InstanceDefaults.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt b/server-core/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt rename to server-core/src/main/kotlin/dev/typetype/server/services/ChannelUrlCanonicalizer.kt diff --git a/server-db/build.gradle.kts b/server-db/build.gradle.kts new file mode 100644 index 00000000..7b90222f --- /dev/null +++ b/server-db/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + kotlin("jvm") +} + +dependencies { + api(project(":server-core")) + api("org.jetbrains.exposed:exposed-core:1.5.0") + api("org.jetbrains.exposed:exposed-jdbc:1.5.0") + api("com.zaxxer:HikariCP:7.1.0") + implementation("org.postgresql:postgresql:42.7.13") + implementation("org.xerial:sqlite-jdbc:3.53.4.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseCollectionMetadataMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseImportedMediaRepairMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseOidcMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabasePrimaryKeyMigrations.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseProfileAccountsMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSessionAuthMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseSubscriptionsCanonicalMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/DatabaseYoutubeRemoteLoginMigration.kt diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/AllowedPlaylistsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedKeywordsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BlockedVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/BugReportsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ChannelNotificationPreferencesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/FavoritesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/HistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationReadItemsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/NotificationStatesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PasswordResetTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PlaylistsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PresenceKeysTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ProfileAccountsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/ProgressTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushDevicesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationBaselinesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationDeliveriesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationEventsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/PushNotificationSeenVideosTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationEventsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedHistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationFeedbackTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingPreferencesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RecommendationOnboardingStateTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SavedPlaylistsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SearchHistoryTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SessionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserAvatarsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserChannelInterestTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UserTopicInterestTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/UsersTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/WatchLaterTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionPairingsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeSessionsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutImportJobsTable.kt diff --git a/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt rename to server-db/src/main/kotlin/dev/typetype/server/db/tables/YoutubeTakeoutPlaylistKeysTable.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index f5aa07a2..4ba30424 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,6 +10,7 @@ dependencyResolutionManagement { include(":server-cache") include(":server-core") +include(":server-db") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull From 784156b6e8aeb8935d175d3503474293768f05c3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:30:24 +0200 Subject: [PATCH 031/137] refactor: extract server-sabr into Gradle module --- build.gradle.kts | 1 + server-sabr/build.gradle.kts | 12 ++++++++++++ .../youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java | 0 .../kotlin/dev/typetype/server/sabr/SabrAdapter.kt | 2 +- .../dev/typetype/server/sabr/SabrExceptions.kt | 4 ++++ .../kotlin/dev/typetype/server/sabr/SabrFormat.kt | 2 +- .../main/kotlin/dev/typetype/server/sabr/SabrInfo.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrMedia.kt | 4 ++-- .../kotlin/dev/typetype/server/sabr/SabrPolicy.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrProfile.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrRange.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrRequest.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrSession.kt | 4 ++-- .../dev/typetype/server/sabr/SabrStreamState.kt | 2 +- .../dev/typetype/server/sabr/SabrTokenProvider.kt | 2 +- settings.gradle.kts | 1 + .../dev/typetype/server/sabr/SabrExceptions.kt | 4 ---- 17 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 server-sabr/build.gradle.kts rename {src => server-sabr/src}/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java (100%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt (98%) create mode 100644 server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt (96%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt (96%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt (94%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt (87%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt (86%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrRange.kt (94%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt (94%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrSession.kt (98%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt (99%) rename {src => server-sabr/src}/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt (73%) delete mode 100644 src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt diff --git a/build.gradle.kts b/build.gradle.kts index 03df8c71..d51e12d2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation(project(":server-cache")) implementation(project(":server-core")) implementation(project(":server-db")) + implementation(project(":server-sabr")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts new file mode 100644 index 00000000..5fa0d34b --- /dev/null +++ b/server-sabr/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + kotlin("jvm") +} + +dependencies { + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java b/server-sabr/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java similarity index 100% rename from src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java rename to server-sabr/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/TypeTypeYoutubeSabrInfoFactory.java diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt index 3ec926d3..30cfc882 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrAdapter.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.TypeTypeYoutubeSabrInf import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrProbe /** The only server entry point for PipePipe's SABR extraction API. */ -internal object SabrAdapter { +object SabrAdapter { fun fetchSabrInfo( videoId: String, profile: YoutubeSabrClientProfile, diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt new file mode 100644 index 00000000..11b4e668 --- /dev/null +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt @@ -0,0 +1,4 @@ +package dev.typetype.server.sabr + +typealias SabrProtocolException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException +typealias SabrRecoverableException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt index 79d46ef0..d933487e 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt @@ -3,7 +3,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as PipeFormat /** TypeType's stable representation of a provider media format. */ -internal class YoutubeSabrFormat internal constructor( +class YoutubeSabrFormat constructor( internal val delegate: PipeFormat, ) { val isAudio: Boolean get() = delegate.isAudio diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt index 68c9785f..f754965f 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as P import java.util.IdentityHashMap /** Provider-independent SABR metadata used by TypeType's orchestration layer. */ -internal class YoutubeSabrInfo internal constructor( +class YoutubeSabrInfo constructor( internal val delegate: PipeInfo, private val formatCache: IdentityHashMap = IdentityHashMap(), ) { diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt index 23b74cee..3b30ee68 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt @@ -4,7 +4,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaHeader as Pip import org.schabi.newpipe.extractor.services.youtube.sabr.SabrMediaSegment as PipeSegment import java.io.InputStream -internal class SabrMediaHeader internal constructor( +class SabrMediaHeader constructor( private val delegate: PipeHeader, ) { val headerId: Int get() = delegate.headerId @@ -27,7 +27,7 @@ internal class SabrMediaHeader internal constructor( fun summarize(): String = delegate.summarize() } -internal class SabrMediaSegment private constructor( +class SabrMediaSegment private constructor( internal val delegate: PipeSegment, ) { val header: SabrMediaHeader = SabrMediaHeader(delegate.header) diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt index 1effadf6..0a916ef3 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrPolicy.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrNextRequestPolicy as PipePolicy -internal class SabrNextRequestPolicy internal constructor( +class SabrNextRequestPolicy constructor( private val delegate: PipePolicy, ) { val targetAudioReadaheadMs: Int get() = delegate.targetAudioReadaheadMs diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt index 0c07023a..3dbd9182 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrProfile.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrClientProfile as PipeProfile -internal enum class YoutubeSabrClientProfile(internal val delegate: PipeProfile) { +enum class YoutubeSabrClientProfile(internal val delegate: PipeProfile) { WEB(PipeProfile.WEB), MWEB(PipeProfile.MWEB), WEB_EMBEDDED(PipeProfile.WEB_EMBEDDED), diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt index 6a8d95ac..528a334b 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRange.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrBufferedRange as PipeRange -internal class SabrBufferedRange internal constructor( +class SabrBufferedRange constructor( val itag: Int, val lastModified: Long, val xtags: String?, diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt index b7685c29..7996c87b 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest as PipeRequest -internal class SabrSegmentRequest private constructor( +class SabrSegmentRequest private constructor( val format: YoutubeSabrFormat, val isInitializationSegment: Boolean, val sequenceNumber: Int, diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt index 587054c4..27e3e081 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrSession.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo as Pip import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrSession as PipeSession import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState -internal class YoutubeSabrSession( +class YoutubeSabrSession( info: YoutubeSabrInfo, audioFormat: YoutubeSabrFormat, videoFormat: YoutubeSabrFormat, @@ -80,7 +80,7 @@ internal class YoutubeSabrSession( fun prepareForForwardJump(request: SabrSegmentRequest, value: Long): Unit = delegate.prepareForForwardJump(request.delegate, value) fun prepareForMissingSegment(request: SabrSegmentRequest): Unit = delegate.prepareForMissingSegment(request.delegate) - internal class DemandResponseResult internal constructor( + class DemandResponseResult internal constructor( private val delegate: PipeSession.DemandResponseResult, ) { val segmentCount: Int get() = delegate.segmentCount diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt index 5689bd70..4712b027 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt @@ -2,7 +2,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState -internal class YoutubeSabrStreamState private constructor( +class YoutubeSabrStreamState private constructor( internal val delegate: PipeState, ) { companion object { diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt rename to server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt index 2d06cfb3..3aa915d9 100644 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrTokenProvider.kt @@ -1,5 +1,5 @@ package dev.typetype.server.sabr -internal fun interface SabrPoTokenProvider { +fun interface SabrPoTokenProvider { fun getPoToken(info: YoutubeSabrInfo, streamState: YoutubeSabrStreamState): ByteArray? } diff --git a/settings.gradle.kts b/settings.gradle.kts index 4ba30424..46dbb9de 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,6 +11,7 @@ dependencyResolutionManagement { include(":server-cache") include(":server-core") include(":server-db") +include(":server-sabr") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull diff --git a/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt b/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt deleted file mode 100644 index fd73c3dc..00000000 --- a/src/main/kotlin/dev/typetype/server/sabr/SabrExceptions.kt +++ /dev/null @@ -1,4 +0,0 @@ -package dev.typetype.server.sabr - -internal typealias SabrProtocolException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrProtocolException -internal typealias SabrRecoverableException = org.schabi.newpipe.extractor.services.youtube.sabr.SabrRecoverableException From 203b33fc370b8cee26ecc5931260eeb5f75604e8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:31:23 +0200 Subject: [PATCH 032/137] refactor: extract server-downloader into Gradle module --- build.gradle.kts | 1 + server-downloader/build.gradle.kts | 13 +++++++++++++ .../typetype/server/downloader/OkHttpDownloader.kt | 0 .../downloader/OkHttpExtractorResponseMapper.kt | 2 +- .../server/downloader/OkHttpStreamingBodyStream.kt | 2 +- .../server/downloader/YoutubeAuthUserContext.kt | 2 +- .../server/downloader/YoutubeProxySelector.kt | 2 +- settings.gradle.kts | 1 + 8 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 server-downloader/build.gradle.kts rename {src => server-downloader/src}/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt (100%) rename {src => server-downloader/src}/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt (92%) rename {src => server-downloader/src}/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt (87%) rename {src => server-downloader/src}/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt (93%) rename {src => server-downloader/src}/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt (97%) diff --git a/build.gradle.kts b/build.gradle.kts index d51e12d2..53231042 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation(project(":server-cache")) implementation(project(":server-core")) implementation(project(":server-db")) + implementation(project(":server-downloader")) implementation(project(":server-sabr")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts new file mode 100644 index 00000000..e3868969 --- /dev/null +++ b/server-downloader/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + kotlin("jvm") +} + +dependencies { + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.squareup.okhttp3:okhttp:5.5.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt index ff13f574..9a47f893 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt @@ -2,7 +2,7 @@ package dev.typetype.server.downloader import org.schabi.newpipe.extractor.downloader.Response -internal object OkHttpExtractorResponseMapper { +object OkHttpExtractorResponseMapper { fun toExtractorResponse(httpResponse: okhttp3.Response): Response { val responseBodyBytes = httpResponse.body.bytes() val responseBody = responseBodyBytes.toString(Charsets.UTF_8) diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt index 6682dd9f..4c94046b 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpStreamingBodyStream.kt @@ -2,7 +2,7 @@ package dev.typetype.server.downloader import java.io.FilterInputStream -internal class OkHttpStreamingBodyStream( +class OkHttpStreamingBodyStream( private val response: okhttp3.Response, ) : FilterInputStream(response.body.byteStream()) { override fun close() { diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt index ad03ff60..a2642c5e 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt @@ -2,7 +2,7 @@ package dev.typetype.server.downloader import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -internal object YoutubeAuthUserContext { +object YoutubeAuthUserContext { @Volatile private var value: Int? = null fun set(authUser: Int?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt rename to server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt index d9016d4b..d80706f4 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeProxySelector.kt @@ -7,7 +7,7 @@ import java.net.ProxySelector import java.net.SocketAddress import java.net.URI -internal class YoutubeProxySelector private constructor( +class YoutubeProxySelector private constructor( private val proxy: Proxy, ) : ProxySelector() { override fun select(uri: URI): List = diff --git a/settings.gradle.kts b/settings.gradle.kts index 46dbb9de..6b88385f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,6 +11,7 @@ dependencyResolutionManagement { include(":server-cache") include(":server-core") include(":server-db") +include(":server-downloader") include(":server-sabr") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") From eed5c6e5d3ef8809a92c1a103ba60cab4316a81c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:36:05 +0200 Subject: [PATCH 033/137] refactor: move downloader and sabr tests to owning modules --- server-downloader/build.gradle.kts | 6 ++++++ .../typetype/server/downloader/YoutubeAuthUserContext.kt | 2 +- .../kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt | 0 .../dev/typetype/server/OkHttpDownloaderStreamingTest.kt | 0 .../kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt | 0 server-sabr/build.gradle.kts | 7 +++++++ .../src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt | 2 +- .../src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt | 2 +- .../src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt | 2 +- .../main/kotlin/dev/typetype/server/sabr/SabrRequest.kt | 2 +- .../kotlin/dev/typetype/server/sabr/SabrStreamState.kt | 2 +- .../dev/typetype/server/sabr/SabrBoundaryContractTest.kt | 0 12 files changed, 19 insertions(+), 6 deletions(-) rename {src => server-downloader/src}/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt (100%) rename {src => server-downloader/src}/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt (100%) rename {src => server-downloader/src}/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt (100%) rename {src => server-sabr/src}/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt (100%) diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts index e3868969..91d348be 100644 --- a/server-downloader/build.gradle.kts +++ b/server-downloader/build.gradle.kts @@ -6,8 +6,14 @@ dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") implementation("com.squareup.okhttp3:okhttp:5.5.0") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt index a2642c5e..2bb6f80e 100644 --- a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/YoutubeAuthUserContext.kt @@ -9,7 +9,7 @@ object YoutubeAuthUserContext { value = authUser } - internal fun headerFor(url: String): String? { + fun headerFor(url: String): String? { val parsed = url.toHttpUrlOrNull() ?: return null val host = parsed.host.lowercase() val isYoutube = host == "youtube.com" || host.endsWith(".youtube.com") diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderCoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/OkHttpDownloaderStreamingTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt rename to server-downloader/src/test/kotlin/dev/typetype/server/YoutubeProxySelectorTest.kt diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts index 5fa0d34b..b33063bc 100644 --- a/server-sabr/build.gradle.kts +++ b/server-sabr/build.gradle.kts @@ -5,8 +5,15 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt index d933487e..30f72996 100644 --- a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrFormat.kt @@ -4,7 +4,7 @@ import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat as P /** TypeType's stable representation of a provider media format. */ class YoutubeSabrFormat constructor( - internal val delegate: PipeFormat, + val delegate: PipeFormat, ) { val isAudio: Boolean get() = delegate.isAudio val isVideo: Boolean get() = delegate.isVideo diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt index f754965f..b6fe26a4 100644 --- a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrInfo.kt @@ -6,7 +6,7 @@ import java.util.IdentityHashMap /** Provider-independent SABR metadata used by TypeType's orchestration layer. */ class YoutubeSabrInfo constructor( - internal val delegate: PipeInfo, + val delegate: PipeInfo, private val formatCache: IdentityHashMap = IdentityHashMap(), ) { val profile: YoutubeSabrClientProfile diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt index 3b30ee68..a023b4fa 100644 --- a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrMedia.kt @@ -28,7 +28,7 @@ class SabrMediaHeader constructor( } class SabrMediaSegment private constructor( - internal val delegate: PipeSegment, + val delegate: PipeSegment, ) { val header: SabrMediaHeader = SabrMediaHeader(delegate.header) val data: ByteArray get() = delegate.data diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt index 7996c87b..6a08db1f 100644 --- a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrRequest.kt @@ -10,7 +10,7 @@ class SabrSegmentRequest private constructor( ) { private val delegateValue: PipeRequest by lazy(LazyThreadSafetyMode.SYNCHRONIZED, delegateFactory) - internal val delegate: PipeRequest + val delegate: PipeRequest get() = delegateValue companion object { diff --git a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt index 4712b027..2db9e245 100644 --- a/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt +++ b/server-sabr/src/main/kotlin/dev/typetype/server/sabr/SabrStreamState.kt @@ -3,7 +3,7 @@ package dev.typetype.server.sabr import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrStreamState as PipeState class YoutubeSabrStreamState private constructor( - internal val delegate: PipeState, + val delegate: PipeState, ) { companion object { const val TRACK_MODE_VIDEO_AND_AUDIO: Int = PipeState.TRACK_MODE_VIDEO_AND_AUDIO diff --git a/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt b/server-sabr/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt rename to server-sabr/src/test/kotlin/dev/typetype/server/sabr/SabrBoundaryContractTest.kt From 93451ab411c966a04f43c440db938149ca9f4f08 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:42:17 +0200 Subject: [PATCH 034/137] refactor: extract server-auth into Gradle module --- build.gradle.kts | 4 ++-- server-auth/build.gradle.kts | 18 ++++++++++++++++++ .../server/services/AccountIdentityService.kt | 0 .../server/services/AuthAccessTokenCodec.kt | 0 .../server/services/AuthCookieHelpers.kt | 0 .../server/services/AuthRefreshTokenHasher.kt | 0 .../typetype/server/services/AuthService.kt | 0 .../server/services/AuthSessionConfig.kt | 0 .../server/services/AuthSessionRefresher.kt | 0 .../server/services/AuthSessionRevoker.kt | 0 .../server/services/AuthSessionStore.kt | 0 .../server/services/AuthSessionTokens.kt | 0 .../server/services/AuthSessionVerifier.kt | 0 .../server/services/AuthTokenIssuer.kt | 0 .../typetype/server/services/AvatarService.kt | 0 .../server/services/OidcAuthService.kt | 0 .../server/services/OidcCallbackSession.kt | 0 .../dev/typetype/server/services/OidcConfig.kt | 0 .../server/services/OidcConfigLoader.kt | 0 .../typetype/server/services/OidcHttpClient.kt | 0 .../server/services/OidcIdTokenValidator.kt | 0 .../typetype/server/services/OidcIdentity.kt | 0 .../server/services/OidcProviderMetadata.kt | 0 .../server/services/OidcRsaKeyResolver.kt | 0 .../dev/typetype/server/services/OidcState.kt | 0 .../typetype/server/services/OidcStateCodec.kt | 0 .../server/services/OidcUserService.kt | 0 .../server/services/PasswordResetService.kt | 0 .../server/services/ProfileAccountService.kt | 0 .../services/ProfileDataDeletionService.kt | 0 .../typetype/server/services/ProfileService.kt | 0 .../server/services/ProfileUpdateResult.kt | 0 settings.gradle.kts | 1 + 33 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 server-auth/build.gradle.kts rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/AvatarService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcAuthService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcConfig.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcIdentity.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcState.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/OidcUserService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/PasswordResetService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/ProfileService.kt (100%) rename {src => server-auth/src}/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 53231042..85c84039 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,6 +25,7 @@ repositories { } dependencies { + implementation(project(":server-auth")) implementation(project(":server-cache")) implementation(project(":server-core")) implementation(project(":server-db")) @@ -58,9 +59,8 @@ dependencies { implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.4.0") - implementation("com.password4j:password4j:1.8.4") - implementation("com.auth0:java-jwt:4.6.1") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("com.password4j:password4j:1.8.4") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") diff --git a/server-auth/build.gradle.kts b/server-auth/build.gradle.kts new file mode 100644 index 00000000..58ed8c60 --- /dev/null +++ b/server-auth/build.gradle.kts @@ -0,0 +1,18 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-db")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + api("com.auth0:java-jwt:4.6.1") + implementation("com.password4j:password4j:1.8.4") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AccountIdentityService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthAccessTokenCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthCookieHelpers.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthRefreshTokenHasher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRefresher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionRevoker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionTokens.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthSessionVerifier.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AuthTokenIssuer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AvatarService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/AvatarService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AvatarService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/AvatarService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcAuthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcCallbackSession.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcConfig.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcConfigLoader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcHttpClient.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdTokenValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcIdentity.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcProviderMetadata.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcRsaKeyResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcState.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcState.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcStateCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OidcUserService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/OidcUserService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/PasswordResetService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileAccountService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileDataDeletionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileService.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileService.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt b/server-auth/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt rename to server-auth/src/main/kotlin/dev/typetype/server/services/ProfileUpdateResult.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 6b88385f..4c33c381 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -8,6 +8,7 @@ dependencyResolutionManagement { } } +include(":server-auth") include(":server-cache") include(":server-core") include(":server-db") From 8e3b6609164adfd8f279238a0ef9bc77dc73a552 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 20:54:04 +0200 Subject: [PATCH 035/137] refactor: extract server-token-gateway into Gradle module --- build.gradle.kts | 1 + server-token-gateway/build.gradle.kts | 31 +++++++++++++++++++ .../AuthenticatedYoutubeVisitorData.kt | 2 +- .../server/services/BoundedExpiringCache.kt | 2 +- .../server/services/SabrPreparedInfo.kt | 4 +-- .../server/services/SabrPreparedSource.kt | 2 +- .../server/services/SabrTokenBundle.kt | 8 ++--- .../server/services/SecretConfigReader.kt | 0 .../server/services/TokenYoutubeSession.kt | 4 +-- .../services/TypetypeTokenSabrTokenClient.kt | 2 +- .../TypetypeTokenYoutubeJavaScriptDecoder.kt | 2 +- .../TypetypeTokenYoutubeSessionClient.kt | 2 +- ...etypeTokenYoutubeSessionPoTokenProvider.kt | 2 +- .../YoutubeRemoteBrowserMessageGuard.kt | 0 .../YoutubeRemoteBrowserReserveResult.kt | 0 .../services/YoutubeRemoteBrowserSession.kt | 0 .../YoutubeRemoteBrowserStartResult.kt | 0 .../YoutubeRemoteBrowserTokenStartRequest.kt | 0 .../YoutubeRemoteBrowserTokenStartResponse.kt | 0 .../services/YoutubeSessionCompleteResult.kt | 0 .../server/services/YoutubeSessionCookie.kt | 0 .../YoutubeSessionCookieNormalizer.kt | 0 .../YoutubeSessionCredentialValidator.kt | 0 .../services/YoutubeSessionCredentials.kt | 0 .../server/services/YoutubeSessionCrypto.kt | 0 .../services/YoutubeSessionPairingCode.kt | 0 .../services/YoutubeSessionPairingStore.kt | 0 .../server/services/YoutubeSessionStatus.kt | 0 .../server/services/YoutubeSessionStore.kt | 0 .../typetype/server/SecretConfigReaderTest.kt | 0 .../TypetypeTokenYoutubeSessionClientTest.kt | 0 ...eTokenYoutubeSessionPoTokenProviderTest.kt | 0 .../services/BoundedExpiringCacheTest.kt | 0 settings.gradle.kts | 1 + 34 files changed, 48 insertions(+), 15 deletions(-) create mode 100644 server-token-gateway/build.gradle.kts rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt (95%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt (98%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt (78%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt (65%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt (91%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt (86%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt (98%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt (98%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt (99%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt (98%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt (100%) rename {src/test/kotlin/dev/typetype/server/services => server-token-gateway/src/test/kotlin/dev/typetype/server}/TypetypeTokenYoutubeSessionClientTest.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 85c84039..75d3b245 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { implementation(project(":server-db")) implementation(project(":server-downloader")) implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) implementation("com.fasterxml.jackson.core:jackson-core") implementation(platform("io.netty:netty-bom:4.2.17.Final")) diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts new file mode 100644 index 00000000..57e97cd8 --- /dev/null +++ b/server-token-gateway/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + kotlin("jvm") +} + +dependencies { + api(project(":server-sabr")) + api(project(":server-core")) + implementation(project(":server-db")) + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.slf4j:slf4j-api:2.0.16") + compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-client-core-jvm:3.5.2") + implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt index 2c2d55db..763c9ad8 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/AuthenticatedYoutubeVisitorData.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper -internal object AuthenticatedYoutubeVisitorData { +object AuthenticatedYoutubeVisitorData { fun fetch( localization: Localization = Localization("en", "US"), contentCountry: ContentCountry = ContentCountry("US"), diff --git a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt index 7a35ef16..b259d320 100644 --- a/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BoundedExpiringCache.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.time.Duration import java.util.LinkedHashMap -internal class BoundedExpiringCache( +class BoundedExpiringCache( private val maxEntries: Int, private val maxWeight: Long = Long.MAX_VALUE, ttl: Duration, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt index d0d661e4..1835ba99 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfo.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrInfo -internal class SabrPreparedInfo( +class SabrPreparedInfo( val info: YoutubeSabrInfo, val initialToken: SabrTokenBundle?, val isLive: Boolean = false, @@ -10,5 +10,5 @@ internal class SabrPreparedInfo( val source: SabrPreparedSource = SabrPreparedSource.PUBLIC, ) -internal fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = +fun SabrPreparedInfo.hasAudioAndVideoFormats(): Boolean = info.formats.any { it.isAudio } && info.formats.any { it.isVideo } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt similarity index 65% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt index 93b526e1..3e7b353f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrPreparedSource.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrPreparedSource { +enum class SabrPreparedSource { PUBLIC, AUTHENTICATED_YOUTUBE, } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt index f4fa83f9..e72ba958 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SabrTokenBundle.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import dev.typetype.server.sabr.YoutubeSabrInfo import java.util.Base64 -internal class SabrTokenBundle( +class SabrTokenBundle( val videoId: String, val visitorBoundPoToken: String, val visitorBoundPoTokenBytes: ByteArray, @@ -63,10 +63,10 @@ internal class SabrTokenBundle( } } -internal fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = +fun SabrTokenBundle.youtubeSessionPoToken(): YoutubeSessionPoToken = YoutubeSessionPoToken(sessionBinding ?: visitorData, sessionBoundPoToken ?: visitorBoundPoToken) -internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = +fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): ByteArray? = takeIf { it.videoId == info.videoId && (it.visitorData == info.visitorData || it.sessionBinding == info.visitorData) @@ -74,4 +74,4 @@ internal fun SabrTokenBundle.streamingPoTokenBytesFor(info: YoutubeSabrInfo): By ?.streamingPoTokenBytes ?.takeIf { it.isNotEmpty() } -internal const val SABR_TOKEN_BINDING_FAILURE = "SABR token does not match session visitorData" +const val SABR_TOKEN_BINDING_FAILURE = "SABR token does not match session visitorData" diff --git a/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/SecretConfigReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt index 713f16a2..2668542a 100644 --- a/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TokenYoutubeSession.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrInfo -internal data class TokenYoutubeSession( +data class TokenYoutubeSession( val info: YoutubeSabrInfo, val token: SabrTokenBundle?, val title: String, @@ -19,7 +19,7 @@ internal data class TokenYoutubeSession( val hlsUrl: String = "", ) -internal fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { +fun TokenYoutubeSession.preparedSabrInfo(): SabrPreparedInfo? { val boundToken = token?.takeIf { it.visitorData == info.visitorData } ?: return null return SabrPreparedInfo(info, boundToken, isLive, isLiveContent) .takeIf(SabrPreparedInfo::hasAudioAndVideoFormats) diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index 6daab550..507229c1 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -8,7 +8,7 @@ import org.json.JSONObject import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal class TypetypeTokenSabrTokenClient( +class TypetypeTokenSabrTokenClient( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) { diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt index 4f7b5cd3..5756317a 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeJavaScriptDecoder.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException import org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptDecoder -internal class TypetypeTokenYoutubeJavaScriptDecoder( +class TypetypeTokenYoutubeJavaScriptDecoder( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) : YoutubeJavaScriptDecoder { diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt index 43074ff1..fd4422fa 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt @@ -18,7 +18,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import kotlin.coroutines.resume -internal class TypetypeTokenYoutubeSessionClient( +class TypetypeTokenYoutubeSessionClient( private val tokenServiceUrl: String, private val client: OkHttpClient = OkHttpClient(), ) { diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt index 15b63405..3ddc8d7f 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionPoTokenProvider.kt @@ -9,7 +9,7 @@ import java.security.MessageDigest import java.util.concurrent.CompletableFuture import java.util.concurrent.ConcurrentHashMap -internal class TypetypeTokenYoutubeSessionPoTokenProvider( +class TypetypeTokenYoutubeSessionPoTokenProvider( private val boundTokenFetcher: (String) -> String?, private val visitorDataFetcher: (Localization, ContentCountry) -> String, private val nowMillis: () -> Long = System::currentTimeMillis, diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserMessageGuard.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserReserveResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSession.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserStartResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartRequest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserTokenStartResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCompleteResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookie.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCookieNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentialValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCredentials.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionCrypto.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingCode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionPairingStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStatus.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStore.kt diff --git a/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/SecretConfigReaderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClientTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionClientTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/TypetypeTokenYoutubeSessionPoTokenProviderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/services/BoundedExpiringCacheTest.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 4c33c381..8467aa6e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,6 +14,7 @@ include(":server-core") include(":server-db") include(":server-downloader") include(":server-sabr") +include(":server-token-gateway") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") .orNull From 59578512b5f92c6fe17df15b8cab90c9152bd77d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 21:15:45 +0200 Subject: [PATCH 036/137] refactor: extract server-portability and server-domain modules --- build.gradle.kts | 2 + server-domain/build.gradle.kts | 16 +++++++ .../typetype/server/services/AccessMode.kt | 11 +++++ .../server/services/NormalizationKeys.kt | 42 +++++++++++++++++++ .../services/SettingsPersistenceMappers.kt | 6 +-- .../SubscriptionFeedCacheInvalidation.kt | 5 +++ .../SubscriptionGroupMembershipCleaner.kt | 2 +- .../services/SubscriptionMutationLock.kt | 2 +- .../server/services/YoutubeTypeTypeMapper.kt | 0 server-portability/build.gradle.kts | 21 ++++++++++ .../portability/FlowPortabilityAdapter.kt | 0 .../portability/FlowPortabilityReader.kt | 2 +- .../portability/FlowPortabilityWriter.kt | 2 +- .../portability/GrayjayPortabilityAdapter.kt | 0 .../portability/GrayjayPortabilityReader.kt | 2 +- .../portability/GrayjayPortabilityWriter.kt | 2 +- .../InvidiousPortabilityAdapter.kt | 0 .../LibreTubePortabilityAdapter.kt | 0 .../portability/LibreTubePortabilityReader.kt | 2 +- .../portability/LibreTubePortabilityWriter.kt | 2 +- .../MaterialiousPortabilityAdapter.kt | 0 .../portability/NewPipeArchiveCapabilities.kt | 4 +- .../portability/NewPipeArchiveDatabase.kt | 2 +- .../NewPipeArchivePortabilityWriter.kt | 2 +- .../portability/NewPipeArchiveRecordWriter.kt | 2 +- .../portability/NewPipeArchiveSchema.kt | 2 +- .../portability/NewPipeArchiveTarget.kt | 2 +- .../NewPipeDatabasePortabilityReader.kt | 2 +- .../portability/NewPipePortabilityAdapter.kt | 0 .../server/portability/NewPipeProvider.kt | 2 +- .../portability/OpmlPortabilityAdapter.kt | 0 .../portability/PipePipePortabilityAdapter.kt | 2 +- .../portability/PipedPortabilityAdapter.kt | 0 .../portability/PipedPortabilityReader.kt | 2 +- .../portability/PipedPortabilityWriter.kt | 2 +- .../server/portability/PortabilityAdapter.kt | 0 .../PortabilityArchiveInventory.kt | 0 .../server/portability/PortabilityDataPort.kt | 0 .../server/portability/PortabilityEngine.kt | 2 +- .../server/portability/PortabilityFormat.kt | 0 .../portability/PortabilityInputFactory.kt | 0 .../server/portability/PortabilityJob.kt | 2 +- .../portability/PortabilityJobModels.kt | 0 .../server/portability/PortabilityJobStore.kt | 2 +- .../server/portability/PortabilityJson.kt | 18 ++++---- .../portability/PortabilityJsonElements.kt | 14 +++---- .../server/portability/PortabilityLimits.kt | 0 .../server/portability/PortabilityProgress.kt | 8 ++-- .../server/portability/PortabilityRecord.kt | 0 .../server/portability/PortabilityRegistry.kt | 4 +- .../server/portability/PortabilitySpool.kt | 0 .../portability/PortabilitySpoolKeys.kt | 4 +- .../portability/PortabilitySpoolSchema.kt | 2 +- .../portability/PortabilityUploadWriter.kt | 0 .../server/portability/PortabilityVideo.kt | 2 +- .../TypeTypePortabilityCoreExport.kt | 2 +- .../TypeTypePortabilityCoreWriter.kt | 4 +- .../TypeTypePortabilityFilterExport.kt | 2 +- .../TypeTypePortabilityFilterWriter.kt | 2 +- .../TypeTypePortabilityHistoryImport.kt | 2 +- .../TypeTypePortabilityLibraryExport.kt | 2 +- .../TypeTypePortabilityLibraryImport.kt | 2 +- .../TypeTypePortabilityLibraryWriter.kt | 2 +- .../TypeTypePortabilitySettingsExport.kt | 2 +- .../TypeTypePortabilitySettingsWriter.kt | 2 +- .../portability/TypeTypePortabilityWriter.kt | 2 +- .../portability/ViewTubePortabilityAdapter.kt | 0 .../portability/ViewTubePortabilityReader.kt | 2 +- .../YoutubeTakeoutCsvPortabilityReader.kt | 2 +- .../YoutubeTakeoutCsvSchemaDetector.kt | 6 +-- .../YoutubeTakeoutHtmlPortabilityReader.kt | 2 +- .../YoutubeTakeoutJsonEntryReader.kt | 6 +-- .../YoutubeTakeoutJsonPortabilityReader.kt | 2 +- .../portability/YoutubeTakeoutMappings.kt | 8 ++-- .../YoutubeTakeoutPortabilityAdapter.kt | 0 .../YoutubeTakeoutActivityClassifier.kt | 0 .../YoutubeTakeoutActivitySignalService.kt | 0 .../services/YoutubeTakeoutCommitPlanner.kt | 0 .../services/YoutubeTakeoutCsvReader.kt | 0 .../services/YoutubeTakeoutDateParser.kt | 0 .../services/YoutubeTakeoutHistoryParser.kt | 4 +- .../YoutubeTakeoutImportJobArchiveStore.kt | 0 .../services/YoutubeTakeoutImportJobFlags.kt | 0 .../YoutubeTakeoutImportJobFlagsStore.kt | 0 .../YoutubeTakeoutImportJobPreviewStore.kt | 0 .../YoutubeTakeoutImportJobReportStore.kt | 0 .../YoutubeTakeoutImportJobStatusStore.kt | 0 .../services/YoutubeTakeoutImportJobStore.kt | 0 .../services/YoutubeTakeoutImportProgress.kt | 2 +- .../services/YoutubeTakeoutIssueService.kt | 0 .../services/YoutubeTakeoutJobRuntimeCache.kt | 0 .../server/services/YoutubeTakeoutLimits.kt | 0 .../services/YoutubeTakeoutPathHints.kt | 0 .../YoutubeTakeoutPlaylistKeyResolver.kt | 0 .../YoutubeTakeoutPlaylistKeyService.kt | 0 .../YoutubeTakeoutPreferenceService.kt | 0 .../services/YoutubeTakeoutPrivacyService.kt | 0 .../services/YoutubeTakeoutRowParser.kt | 0 .../services/YoutubeTakeoutSchemaHints.kt | 0 .../services/YoutubeTakeoutSchemaTerms.kt | 2 +- .../services/YoutubeTakeoutSystemPlaylist.kt | 0 .../services/YoutubeTakeoutTextNormalizer.kt | 0 .../services/YoutubeTakeoutUnavailableItem.kt | 2 +- .../services/YoutubeTakeoutUploadWriter.kt | 0 .../services/YoutubeTakeoutZipScanResult.kt | 0 .../services/YoutubeTakeoutZipScanner.kt | 0 settings.gradle.kts | 2 + .../dev/typetype/server/ServiceRegistry.kt | 4 +- .../TypeTypePortabilitySettingsImport.kt | 2 +- .../typetype/server/services/AccessMode.kt | 11 ----- .../server/services/AllowedChannelsService.kt | 31 -------------- .../services/AllowedPlaylistsService.kt | 5 --- .../server/services/BlockedService.kt | 3 -- .../SubscriptionFeedCacheInvalidator.kt | 8 ++-- ...iptionFeedCacheInvalidationPipePipeTest.kt | 4 +- ...onFeedCacheInvalidationPipePipeUnitTest.kt | 4 +- ...ionFeedCacheInvalidationTakeoutUnitTest.kt | 4 +- 117 files changed, 195 insertions(+), 146 deletions(-) create mode 100644 server-domain/build.gradle.kts create mode 100644 server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt create mode 100644 server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt (97%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt (77%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt (93%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt (91%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt (100%) create mode 100644 server-portability/build.gradle.kts rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt (93%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt (95%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt (85%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt (95%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt (96%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt (77%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt (56%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt (93%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt (95%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt (63%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt (91%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt (93%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt (92%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt (97%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt (96%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt (99%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt (85%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt (94%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt (93%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt (89%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt (100%) delete mode 100644 src/main/kotlin/dev/typetype/server/services/AccessMode.kt diff --git a/build.gradle.kts b/build.gradle.kts index 75d3b245..6a81ea6f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,6 +30,8 @@ dependencies { implementation(project(":server-core")) implementation(project(":server-db")) implementation(project(":server-downloader")) + implementation(project(":server-domain")) + implementation(project(":server-portability")) implementation(project(":server-sabr")) implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts new file mode 100644 index 00000000..5a009688 --- /dev/null +++ b/server-domain/build.gradle.kts @@ -0,0 +1,16 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt new file mode 100644 index 00000000..30711273 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/AccessMode.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.services + +const val ACCESS_MODE_UNRESTRICTED = "unrestricted" +const val ACCESS_MODE_ALLOW_LIST = "allow_list" +const val ALLOW_SCOPE_USER = "user" +const val ALLOW_SCOPE_GLOBAL = "global" + +fun String.toAccessMode(): String = when (this) { + ACCESS_MODE_ALLOW_LIST -> ACCESS_MODE_ALLOW_LIST + else -> ACCESS_MODE_UNRESTRICTED +} diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt new file mode 100644 index 00000000..d65424f5 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/NormalizationKeys.kt @@ -0,0 +1,42 @@ +package dev.typetype.server.services + +import java.text.Normalizer +import java.net.URI + +fun normalizeChannelKey(value: String): String = value.trim() + .substringBefore('#') + .substringBefore('?') + .removeSuffix("/") + .replace("http://", "https://") + .replace( + Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), + "https://youtube.com", + ) + .withoutYoutubeTab() + +private fun String.withoutYoutubeTab(): String { + val uri = runCatching { URI(this) }.getOrNull() ?: return this + if (!uri.host.equals("youtube.com", ignoreCase = true)) return this + val segments = uri.path.split('/').filter(String::isNotBlank) + if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this + val path = "/${segments.dropLast(1).joinToString("/")}" + return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() +} + +private val YOUTUBE_CHANNEL_TABS = setOf( + "featured", + "videos", + "shorts", + "streams", + "playlists", + "community", + "about", +) + +fun normalizePlaylistKey(value: String): String = value.trim() + .substringBefore('#') + .removeSuffix("/") + .replace("http://", "https://") + +fun normalizeBlockedKeyword(value: String): String = + Normalizer.normalize(value, Normalizer.Form.NFKC).trim().lowercase() diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 8963c806..8e7ad03b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -16,7 +16,7 @@ import org.jetbrains.exposed.v1.core.statements.UpdateBuilder private val SPONSOR_BLOCK_CATEGORY_ACTIONS_SERIALIZER = MapSerializer(String.serializer(), SponsorBlockMode.serializer()) private val CAPTION_STYLES_SERIALIZER = CaptionStylesItem.serializer() -internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( +fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultService = this[SettingsTable.defaultService], defaultQuality = this[SettingsTable.defaultQuality], defaultPlaybackSpeed = this[SettingsTable.defaultPlaybackSpeed], @@ -57,7 +57,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( accessMode = this[SettingsTable.accessMode].toAccessMode(), ) -internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { +fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultService] = settings.defaultService this[SettingsTable.defaultQuality] = settings.defaultQuality this[SettingsTable.defaultPlaybackSpeed] = settings.defaultPlaybackSpeed @@ -100,7 +100,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.accessModeAdminManagedAt] = 0 } -internal fun SettingsItem.normalized(): SettingsItem = copy( +fun SettingsItem.normalized(): SettingsItem = copy( defaultLandingPage = defaultLandingPage.ifBlank { "home" }, defaultPlaybackSpeed = defaultPlaybackSpeed.takeIf { it.isFinite() } ?.coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED) ?: 1.0, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt index 410755b6..44fc3ff2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidation.kt @@ -1,5 +1,10 @@ package dev.typetype.server.services +interface SubscriptionFeedCacheInvalidator { + suspend fun invalidate(userId: String) + suspend fun awaitRefresh(userId: String) +} + object SubscriptionFeedCacheInvalidation { @Volatile private var invalidator: SubscriptionFeedCacheInvalidator? = null diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt index d78f7833..55a1368f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt @@ -6,7 +6,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.jdbc.deleteWhere -internal object SubscriptionGroupMembershipCleaner { +object SubscriptionGroupMembershipCleaner { fun retain(userId: String, channelUrls: Collection) { val retained = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) SubscriptionGroupMembershipsTable.deleteWhere { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt index 8320b1ab..50b3c3b3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager -internal object SubscriptionMutationLock { +object SubscriptionMutationLock { fun acquire(userId: String) { val userKey = userId.hashCode() and Int.MAX_VALUE TransactionManager.current().exec( diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeTypeTypeMapper.kt diff --git a/server-portability/build.gradle.kts b/server-portability/build.gradle.kts new file mode 100644 index 00000000..2b6b19b7 --- /dev/null +++ b/server-portability/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-utils-jvm:3.5.2") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("com.fasterxml.jackson.core:jackson-databind:2.22.2") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt index ecc999e6..046058f4 100644 --- a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import kotlinx.serialization.json.jsonObject -internal object FlowPortabilityReader { +object FlowPortabilityReader { fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) = sink.readFlowArray(PortabilityCategory.SUBSCRIPTIONS, parser, token) { item -> val id = item.string("channelId") diff --git a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt index f087b3bf..2f2cc4ea 100644 --- a/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/FlowPortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object FlowPortabilityWriter { +object FlowPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt index 29fba1d8..3c2b6bd6 100644 --- a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import java.util.zip.ZipFile -internal object GrayjayPortabilityReader { +object GrayjayPortabilityReader { fun read(zip: ZipFile, sink: PortabilityRecordSink) { readStrings(zip, "stores/subscriptions") { readSubscription(it, sink) } readStrings(zip, "stores/subscription_groups") { readGroup(it, sink) } diff --git a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt index da23f917..621587a3 100644 --- a/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/GrayjayPortabilityWriter.kt @@ -6,7 +6,7 @@ import java.util.UUID import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -internal object GrayjayPortabilityWriter { +object GrayjayPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { ZipOutputStream(output.buffered()).use { zip -> writeObjectEntry(zip, "exportInfo", mapOf("version" to "1")) diff --git a/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt index afb7c4f2..c16a20bc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityReader.kt @@ -5,7 +5,7 @@ import com.fasterxml.jackson.core.JsonToken import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.jsonObject -internal object LibreTubePortabilityReader { +object LibreTubePortabilityReader { fun history(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken) { sink.readArray(PortabilityCategory.HISTORY, parser, token) { element -> val item = element.jsonObject diff --git a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt index f3eb1915..568fdd9e 100644 --- a/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/LibreTubePortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object LibreTubePortabilityWriter { +object LibreTubePortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt index 3c99df75..17a517fc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveCapabilities.kt @@ -1,6 +1,6 @@ package dev.typetype.server.portability -internal fun newPipeArchiveCapabilities(): Set = +fun newPipeArchiveCapabilities(): Set = archiveImportCapabilities().mapTo(linkedSetOf()) { capability -> PortabilityCapability( capability.category, @@ -12,7 +12,7 @@ internal fun newPipeArchiveCapabilities(): Set = ) } -internal fun newPipeArchiveExportIssues( +fun newPipeArchiveExportIssues( source: PortabilityRecordSource, categories: Set, target: NewPipeArchiveTarget, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt index ad822179..d0ea698a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveDatabase.kt @@ -7,7 +7,7 @@ import java.sql.Connection import java.sql.DriverManager import java.util.zip.ZipFile -internal object NewPipeArchiveDatabase { +object NewPipeArchiveDatabase { private val databaseNames = setOf("newpipe.db", "pipepipe.db") fun userVersion(input: PortabilityInput): Int? { diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt index 96d2b206..ae448eb7 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchivePortabilityWriter.kt @@ -6,7 +6,7 @@ import java.sql.DriverManager import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -internal object NewPipeArchivePortabilityWriter { +object NewPipeArchivePortabilityWriter { fun write( source: PortabilityRecordSource, output: OutputStream, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt index 8c08762a..d5835e3f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveRecordWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.sql.Connection -internal class NewPipeArchiveRecordWriter( +class NewPipeArchiveRecordWriter( private val db: Connection, private val target: NewPipeArchiveTarget, ) { diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt index e83b543f..336dcca1 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveSchema.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.sql.Connection -internal object NewPipeArchiveSchema { +object NewPipeArchiveSchema { fun create(db: Connection, target: NewPipeArchiveTarget) { db.createStatement().use { statement -> statement.execute("PRAGMA foreign_keys = ON") diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt index 61bf8305..49794bba 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeArchiveTarget.kt @@ -1,6 +1,6 @@ package dev.typetype.server.portability -internal enum class NewPipeArchiveTarget( +enum class NewPipeArchiveTarget( val databaseVersion: Int, val identityHash: String, val pipePipe: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt index 87d057f8..67d3c6dc 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeDatabasePortabilityReader.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import java.sql.Connection import java.sql.ResultSet -internal object NewPipeDatabasePortabilityReader { +object NewPipeDatabasePortabilityReader { fun read(connection: Connection, sink: PortabilityRecordSink) { subscriptions(connection, sink) groups(connection, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt index 084cf6bc..742793b0 100644 --- a/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/NewPipeProvider.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import java.net.URI -internal object NewPipeProvider { +object NewPipeProvider { fun serviceId(url: String): Int? { val host = runCatching { URI(url.trim()).host?.lowercase() }.getOrNull() ?: return null return when { diff --git a/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt index e5bb46fb..1ea4f22f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipePipePortabilityAdapter.kt @@ -36,7 +36,7 @@ class PipePipePortabilityAdapter : PortabilityAdapter { } } -internal fun archiveImportCapabilities(): Set = setOf( +fun archiveImportCapabilities(): Set = setOf( PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.SUBSCRIPTION_GROUPS, PortabilityCategory.HISTORY, diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt index 30703bf9..c0c4d449 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityReader.kt @@ -6,7 +6,7 @@ import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.jsonObject -internal object PipedPortabilityReader { +object PipedPortabilityReader { fun subscriptions(parser: JsonParser, sink: PortabilityRecordSink, token: JsonToken = JsonToken.START_ARRAY) { require(token == JsonToken.START_ARRAY) { "Invalid Piped subscriptions" } sink.markCategory(PortabilityCategory.SUBSCRIPTIONS) diff --git a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt index 304e6cde..d6d65011 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PipedPortabilityWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator import java.io.OutputStream -internal object PipedPortabilityWriter { +object PipedPortabilityWriter { fun write(source: PortabilityRecordSource, output: OutputStream, categories: Set) { PortabilityJsonFactory.createGenerator(output).use { json -> json.writeStartObject() diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityArchiveInventory.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityDataPort.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index bb893493..790caec8 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -10,7 +10,7 @@ import kotlinx.coroutines.withContext import java.nio.file.Files import java.nio.file.Path -class PortabilityEngine internal constructor( +class PortabilityEngine constructor( private val registry: PortabilityRegistry, private val dataPort: PortabilityDataPort, private val store: PortabilityJobStore, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityFormat.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityInputFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt index 100ecf63..cd978274 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -6,7 +6,7 @@ import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.atomic.AtomicReference -internal class PortabilityJob( +class PortabilityJob( val id: String, val ownerId: String, val kind: PortabilityJobKind, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt index feb3eae2..d837db37 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobStore.kt @@ -5,7 +5,7 @@ import java.nio.file.Path import java.util.UUID import java.util.concurrent.ConcurrentHashMap -internal class PortabilityJobStore( +class PortabilityJobStore( private val root: Path, private val clock: () -> Long = System::currentTimeMillis, private val retentionMs: Long = DEFAULT_RETENTION_MS, diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt index 8e0f00b1..aaf6271d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJson.kt @@ -10,7 +10,7 @@ import kotlinx.serialization.json.JsonElement import java.nio.file.Files import java.io.ByteArrayOutputStream -internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() +val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() .streamReadConstraints( StreamReadConstraints.builder() .maxNestingDepth(100) @@ -21,22 +21,22 @@ internal val PortabilityJsonFactory: JsonFactory = JsonFactory.builder() .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) .build() -internal inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = +inline fun PortabilityInput.withJsonParser(block: (JsonParser) -> T): T = Files.newInputStream(path).buffered().use { input -> PortabilityJsonFactory.createParser(input).use(block) } -internal fun JsonParser.requireObject() { +fun JsonParser.requireObject() { require(nextToken() == JsonToken.START_OBJECT) { "Backup root must be a JSON object" } } -internal fun JsonParser.textOrEmpty(): String = +fun JsonParser.textOrEmpty(): String = if (currentToken().isScalarValue) valueAsString.orEmpty() else "" -internal fun JsonParser.longOrZero(): Long = +fun JsonParser.longOrZero(): Long = if (currentToken().isNumeric) longValue else valueAsString?.toLongOrNull() ?: 0L -internal fun JsonParser.readJsonElement(): JsonElement { +fun JsonParser.readJsonElement(): JsonElement { val bytes = ByteArrayOutputStream() PortabilityJsonFactory.createGenerator(bytes).use { generator -> generator.copyCurrentStructure(this) @@ -45,7 +45,7 @@ internal fun JsonParser.readJsonElement(): JsonElement { return CacheJson.parseToJsonElement(bytes.toString(Charsets.UTF_8)) } -internal fun youtubeChannelUrl(value: String): String { +fun youtubeChannelUrl(value: String): String { val trimmed = value.trim() return when { trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed @@ -54,7 +54,7 @@ internal fun youtubeChannelUrl(value: String): String { } } -internal fun youtubeVideoUrl(value: String): String { +fun youtubeVideoUrl(value: String): String { val trimmed = value.trim() return when { trimmed.startsWith("http://") || trimmed.startsWith("https://") -> trimmed @@ -63,6 +63,6 @@ internal fun youtubeVideoUrl(value: String): String { } } -internal fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { +fun youtubeId(value: String): String = value.substringAfterLast('/').substringAfterLast('=').takeWhile { it != '&' && it != '?' && it != '#' } diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt similarity index 56% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt index a6d0dc0d..28836960 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJsonElements.kt @@ -8,24 +8,24 @@ import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonPrimitive -internal fun JsonObject.string(name: String): String = get(name)?.let { element -> +fun JsonObject.string(name: String): String = get(name)?.let { element -> runCatching { element.jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") }.orEmpty() -internal fun JsonObject.long(name: String): Long = get(name)?.let { element -> +fun JsonObject.long(name: String): Long = get(name)?.let { element -> runCatching { element.jsonPrimitive.contentOrNull?.toLongOrNull() ?: 0L }.getOrDefault(0L) } ?: 0L -internal fun JsonObject.int(name: String): Int = long(name).toInt() +fun JsonObject.int(name: String): Int = long(name).toInt() -internal fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> +fun JsonObject.array(name: String): JsonArray = get(name)?.let { element -> runCatching { element.jsonArray }.getOrNull() } ?: JsonArray(emptyList()) -internal fun JsonElement.stringValue(): String = runCatching { +fun JsonElement.stringValue(): String = runCatching { jsonPrimitive.contentOrNull.orEmpty() }.getOrDefault("") -internal fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject +fun JsonElement.objectOrNull(): JsonObject? = this as? JsonObject -internal fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive +fun JsonObject.primitiveOrNull(name: String): JsonPrimitive? = get(name) as? JsonPrimitive diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt index 5f92e28b..bc8c88eb 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.CancellationException import java.io.FilterOutputStream import java.io.OutputStream -internal class PortabilityProgressReporter( +class PortabilityProgressReporter( private val job: PortabilityJob, private val phase: PortabilityProgressPhase, private val unit: PortabilityProgressUnit, @@ -41,11 +41,11 @@ internal class PortabilityProgressReporter( } } -internal fun portabilityProgressInterval(total: Long?): Long = total +fun portabilityProgressInterval(total: Long?): Long = total ?.let { (it / 100L).coerceIn(1L, 100L) } ?: 100L -internal class ProgressRecordSink( +class ProgressRecordSink( private val delegate: PortabilityRecordSink, private val progress: PortabilityProgressReporter, ) : PortabilityRecordSink { @@ -78,7 +78,7 @@ internal class ProgressRecordSink( (delegate as? PortabilityRecordSource)?.counts()?.get(category) ?: 0L } -internal class ProgressOutputStream( +class ProgressOutputStream( output: OutputStream, private val progress: PortabilityProgressReporter, ) : FilterOutputStream(output) { diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRecord.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt index 7ffa01a1..668991fa 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityRegistry.kt @@ -63,7 +63,7 @@ class PortabilityFormatMismatchException(format: PortabilityFormat) : class PortabilityUploadTooLargeException : PortabilityContractException("portability_upload_too_large", "Backup exceeds the upload limit") -internal fun portabilityErrorCode(error: Exception): String = when (error) { +fun portabilityErrorCode(error: Exception): String = when (error) { is PortabilityContractException -> error.code is PortabilityJobNotFoundException -> "portability_job_not_found" is IllegalStateException -> "portability_invalid_state" @@ -71,7 +71,7 @@ internal fun portabilityErrorCode(error: Exception): String = when (error) { else -> "portability_failed" } -internal fun portabilityErrorMessage(error: Exception): String = when (error) { +fun portabilityErrorMessage(error: Exception): String = when (error) { is PortabilityContractException, is IllegalArgumentException -> error.message ?: "Invalid portability data" else -> "Portability operation failed" diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt similarity index 63% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt index de0ff23d..9d3f4bf4 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolKeys.kt @@ -3,10 +3,10 @@ package dev.typetype.server.portability import java.security.MessageDigest import java.util.HexFormat -internal fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") +fun portabilityStableHash(value: String): String = MessageDigest.getInstance("SHA-256") .digest(value.toByteArray(Charsets.UTF_8)) .let(HexFormat.of()::formatHex) -internal fun portabilityCategoryByWireName(value: String): PortabilityCategory = +fun portabilityCategoryByWireName(value: String): PortabilityCategory = PortabilityCategory.entries.firstOrNull { it.wireName == value } ?: error("Unknown portability category: $value") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt index 7f81f08e..ccedeb7a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpoolSchema.kt @@ -5,7 +5,7 @@ import java.nio.file.Path import java.sql.Connection import java.sql.DriverManager -internal object PortabilitySpoolSchema { +object PortabilitySpoolSchema { fun create(directory: Path, factory: (Path, Connection) -> T): T { Files.createDirectories(directory) val path = Files.createTempFile(directory, "portability-", ".sqlite") diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt index c71f3a18..2c5c8421 100644 --- a/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityVideo.kt @@ -15,7 +15,7 @@ data class PortabilityVideo( val publishedAt: Long = -1L, ) -internal fun PortabilityVideo.normalized(): PortabilityVideo = copy( +fun PortabilityVideo.normalized(): PortabilityVideo = copy( url = url.trim(), title = title.trim(), thumbnailUrl = thumbnailUrl.trim(), diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt index 08c313c3..ce9b6b4f 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreExport.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityCoreExport { +object TypeTypePortabilityCoreExport { fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { when (category) { PortabilityCategory.SUBSCRIPTIONS -> subscriptions(userId, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt index 6d133576..0b095fc0 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityCoreWriter { +object TypeTypePortabilityCoreWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, @@ -109,7 +109,7 @@ internal object TypeTypePortabilityCoreWriter { } } -internal fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { +fun JsonGenerator.writeVideoFields(video: PortabilityVideo) { writeStringField("title", video.title) writeStringField("thumbnail", video.thumbnailUrl) writeNumberField("duration", video.durationSeconds) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt index a575fa7b..2cb96b9d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterExport.kt @@ -11,7 +11,7 @@ import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityFilterExport { +object TypeTypePortabilityFilterExport { private const val USER_SCOPE = "user" fun write(userId: String, sink: PortabilityRecordSink) { diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt index 5e37bb41..3d51d893 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityFilterWriter { +object TypeTypePortabilityFilterWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt index ca92e360..17aa9d3d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImport.kt @@ -7,7 +7,7 @@ import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.select import java.util.UUID -internal object TypeTypePortabilityHistoryImport { +object TypeTypePortabilityHistoryImport { private const val BATCH_SIZE = 500 fun write( diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt index fb7d4dd3..805a0682 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryExport.kt @@ -9,7 +9,7 @@ import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilityLibraryExport { +object TypeTypePortabilityLibraryExport { fun write(userId: String, category: PortabilityCategory, sink: PortabilityRecordSink) { when (category) { PortabilityCategory.WATCH_LATER -> watchLater(userId, sink) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt index cd9a4ed0..46de4f9d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryImport.kt @@ -13,7 +13,7 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import java.nio.charset.StandardCharsets import java.util.UUID -internal object TypeTypePortabilityLibraryImport { +object TypeTypePortabilityLibraryImport { fun write( userId: String, category: PortabilityCategory, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt index 31d2946f..2139dd39 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityLibraryWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilityLibraryWriter { +object TypeTypePortabilityLibraryWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt index f42d596b..43ffc01d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsExport.kt @@ -9,7 +9,7 @@ import kotlinx.serialization.json.jsonObject import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal object TypeTypePortabilitySettingsExport { +object TypeTypePortabilitySettingsExport { fun write(userId: String, sink: PortabilityRecordSink) { val settings = SettingsTable.selectAll().where { SettingsTable.userId eq userId } .singleOrNull()?.toSettingsItem() ?: SettingsItem() diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt index 0ca1d0f6..91cd9bb5 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsWriter.kt @@ -2,7 +2,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonGenerator -internal object TypeTypePortabilitySettingsWriter { +object TypeTypePortabilitySettingsWriter { fun write( json: JsonGenerator, source: PortabilityRecordSource, diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt index 66fa7b75..5d4ea0d7 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityWriter.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.TYPE_TYPE_BACKUP_FORMAT import dev.typetype.server.models.TYPE_TYPE_BACKUP_VERSION import java.io.OutputStream -internal object TypeTypePortabilityWriter { +object TypeTypePortabilityWriter { fun write( source: PortabilityRecordSource, output: OutputStream, diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt index 337594cc..642ed24d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/ViewTubePortabilityReader.kt @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken import java.time.Instant -internal object ViewTubePortabilityReader { +object ViewTubePortabilityReader { fun read(parser: JsonParser, sink: PortabilityRecordSink) { parser.requireObject() while (parser.nextToken() != JsonToken.END_OBJECT) { diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index 7b2e7249..08e452d4 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -10,7 +10,7 @@ import java.io.InputStreamReader import java.util.zip.ZipEntry import java.util.zip.ZipFile -internal object YoutubeTakeoutCsvPortabilityReader { +object YoutubeTakeoutCsvPortabilityReader { private const val PLAYLIST_LOOKUP = "youtube-takeout-playlist" fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt index edca04f7..80d41e08 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetector.kt @@ -9,20 +9,20 @@ import java.io.InputStreamReader import java.util.zip.ZipEntry import java.util.zip.ZipFile -internal enum class YoutubeTakeoutCsvKind { +enum class YoutubeTakeoutCsvKind { PLAYLIST_MANIFEST, SUBSCRIPTIONS, PLAYLIST_CONTENT, OTHER, } -internal data class YoutubeTakeoutCsvClassification( +data class YoutubeTakeoutCsvClassification( val entry: ZipEntry, val kind: YoutubeTakeoutCsvKind, val maybePortable: Boolean, ) -internal object YoutubeTakeoutCsvSchemaDetector { +object YoutubeTakeoutCsvSchemaDetector { fun classify(zip: ZipFile, entries: List): List = entries.asSequence() .filter { it.name.endsWith(".csv", ignoreCase = true) && "youtube" in it.name.lowercase() } .map { entry -> classify(zip, entry) } diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt index c36811df..54cdc1e2 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReader.kt @@ -7,7 +7,7 @@ import java.io.Reader import java.util.zip.ZipEntry import java.util.zip.ZipFile -internal object YoutubeTakeoutHtmlPortabilityReader { +object YoutubeTakeoutHtmlPortabilityReader { fun read(zip: ZipFile, entries: List, sink: PortabilityRecordSink) { entries.asSequence().filter(::isYoutubeHtml).forEach { entry -> zip.getInputStream(entry).bufferedReader().use { reader -> diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt index f184d338..c15c867a 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonEntryReader.kt @@ -3,7 +3,7 @@ package dev.typetype.server.portability import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.JsonToken -internal object YoutubeTakeoutJsonEntryReader { +object YoutubeTakeoutJsonEntryReader { fun read(parser: JsonParser): YoutubeTakeoutJsonEntry { var title = "" var titleUrl = "" @@ -93,7 +93,7 @@ internal object YoutubeTakeoutJsonEntryReader { } } -internal data class YoutubeTakeoutJsonEntry( +data class YoutubeTakeoutJsonEntry( val title: String, val titleUrl: String, val time: String, @@ -105,4 +105,4 @@ internal data class YoutubeTakeoutJsonEntry( val details: List, ) -internal data class YoutubeTakeoutJsonSubtitle(val name: String, val url: String) +data class YoutubeTakeoutJsonSubtitle(val name: String, val url: String) diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt index c338fbfc..af42b6a3 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutJsonPortabilityReader.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.YoutubeTakeoutPathHints import dev.typetype.server.services.YoutubeTakeoutTextNormalizer import java.io.InputStream -internal object YoutubeTakeoutJsonPortabilityReader { +object YoutubeTakeoutJsonPortabilityReader { private val videoUrlRegex = Regex( """(?:https?://|URLs://)(?:www\.|music\.)?youtube\.com/(?:watch\?(?:[^#\s]*&)?v=|shorts/|live/)[A-Za-z0-9_-]{6,}|https?://youtu\.be/[A-Za-z0-9_-]{6,}""", RegexOption.IGNORE_CASE, diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt index b335dc2b..1fa7531d 100644 --- a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutMappings.kt @@ -6,14 +6,14 @@ import dev.typetype.server.models.PlaylistVideoItem import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.YoutubeTypeTypeMapper -internal fun SubscriptionItem.toPortability() = PortabilitySubscription( +fun SubscriptionItem.toPortability() = PortabilitySubscription( channelUrl = channelUrl, name = name, avatarUrl = avatarUrl, subscribedAt = subscribedAt, ) -internal fun HistoryItem.toPortability() = PortabilityHistory( +fun HistoryItem.toPortability() = PortabilityHistory( video = PortabilityVideo( url = url, title = title, @@ -27,7 +27,7 @@ internal fun HistoryItem.toPortability() = PortabilityHistory( positionSeconds = progress, ) -internal fun FavoriteItem.toPortability() = PortabilityFavorite( +fun FavoriteItem.toPortability() = PortabilityFavorite( video = PortabilityVideo( url = videoUrl, title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(videoUrl) }, @@ -42,7 +42,7 @@ internal fun FavoriteItem.toPortability() = PortabilityFavorite( favoritedAt = favoritedAt, ) -internal fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( +fun PlaylistVideoItem.toPortabilityVideo() = PortabilityVideo( url = url, title = title.ifBlank { YoutubeTypeTypeMapper.titleForUrl(url) }, thumbnailUrl = thumbnail.ifBlank { YoutubeTypeTypeMapper.thumbnailForUrl(url) }, diff --git a/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivityClassifier.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutActivitySignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCommitPlanner.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutCsvReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutDateParser.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt index 51d06973..4dcee75e 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutHistoryParser.kt @@ -15,7 +15,7 @@ object YoutubeTakeoutHistoryParser { fun parse(html: String, requireWatchedMarker: Boolean = true): List = parseWithDiagnostics(html, requireWatchedMarker).items - internal fun parseWithDiagnostics(html: String, requireWatchedMarker: Boolean = true): YoutubeTakeoutHistoryParseResult { + fun parseWithDiagnostics(html: String, requireWatchedMarker: Boolean = true): YoutubeTakeoutHistoryParseResult { val resolvedHtml = html.replace("\u00a0", " ") var invalidDates = 0L val items = videoLinkRegex.findAll(resolvedHtml).mapNotNull { match -> @@ -76,7 +76,7 @@ object YoutubeTakeoutHistoryParser { private const val WATCHED_PREFIX_CHARS = 160 } -internal data class YoutubeTakeoutHistoryParseResult( +data class YoutubeTakeoutHistoryParseResult( val items: List, val invalidDates: Long, ) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobArchiveStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlags.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobFlagsStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobPreviewStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobReportStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStatusStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt index 94bf42cf..692bf522 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportProgress.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import kotlinx.coroutines.channels.Channel import java.util.concurrent.atomic.AtomicInteger -internal class YoutubeTakeoutImportProgress(private val total: Long) { +class YoutubeTakeoutImportProgress(private val total: Long) { private val updates = Channel(Channel.UNLIMITED) private val published = AtomicInteger(-1) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutIssueService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutJobRuntimeCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPathHints.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPlaylistKeyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreferenceService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPrivacyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutRowParser.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaHints.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt index 699c6ac1..d20e8dbc 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSchemaTerms.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object YoutubeTakeoutSchemaTerms { +object YoutubeTakeoutSchemaTerms { val titleWords = setOf( "title", "titles", "titre", "titres", "titulo", "titulos", "baslik", "baslık", "başlık", "basligi", "baslıgı", "başlığı", "adi", "adı", "nombre", "nome", "nom", "name", "names", diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSystemPlaylist.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutTextNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt index 808d72b7..7740a3c9 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUnavailableItem.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object YoutubeTakeoutUnavailableItem { +object YoutubeTakeoutUnavailableItem { fun matches(title: String): Boolean = YoutubeTakeoutTextNormalizer.normalize(title) in TITLES diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutZipScanner.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 8467aa6e..d0468309 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -13,6 +13,8 @@ include(":server-cache") include(":server-core") include(":server-db") include(":server-downloader") +include(":server-domain") +include(":server-portability") include(":server-sabr") include(":server-token-gateway") diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 5ca5063c..1ca8c71b 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -34,7 +34,7 @@ import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.TypeTypeBackupService import dev.typetype.server.services.UserVideoMetadataRepairService import dev.typetype.server.services.VideoMetadataResolver @@ -102,7 +102,7 @@ internal class ServiceRegistry( ) init { SubscriptionFeedCacheInvalidation.configure( - SubscriptionFeedCacheInvalidator(cache, subscriptionFeedService), + SubscriptionFeedCacheInvalidatorImpl(cache, subscriptionFeedService), ) } val notificationsService = NotificationsService(subscriptionFeedService) diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt index 5f457278..c45f91fb 100644 --- a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt +++ b/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt @@ -4,7 +4,7 @@ import dev.typetype.server.cache.CacheJson import dev.typetype.server.models.SettingsItem import dev.typetype.server.services.TypeTypeBackupLibraryRestore -internal object TypeTypePortabilitySettingsImport { +object TypeTypePortabilitySettingsImport { fun write(userId: String, source: PortabilityRecordSource, onRecord: () -> Unit): Long { var count = 0L source.forEach(PortabilityCategory.SETTINGS) { record -> diff --git a/src/main/kotlin/dev/typetype/server/services/AccessMode.kt b/src/main/kotlin/dev/typetype/server/services/AccessMode.kt deleted file mode 100644 index a0fbd39b..00000000 --- a/src/main/kotlin/dev/typetype/server/services/AccessMode.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dev.typetype.server.services - -internal const val ACCESS_MODE_UNRESTRICTED = "unrestricted" -internal const val ACCESS_MODE_ALLOW_LIST = "allow_list" -internal const val ALLOW_SCOPE_USER = "user" -internal const val ALLOW_SCOPE_GLOBAL = "global" - -internal fun String.toAccessMode(): String = when (this) { - ACCESS_MODE_ALLOW_LIST -> ACCESS_MODE_ALLOW_LIST - else -> ACCESS_MODE_UNRESTRICTED -} diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt index 222bb64a..2c31135d 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt @@ -11,7 +11,6 @@ import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll -import java.net.URI class AllowedChannelsService { suspend fun getChannels(userId: String): List = DatabaseFactory.query { @@ -81,33 +80,3 @@ private fun toAllowedChannelItem(row: ResultRow): AllowedChannelItem = AllowedCh allowedAt = row[AllowedChannelsTable.allowedAt], global = row[AllowedChannelsTable.scope] == ALLOW_SCOPE_GLOBAL, ) - -internal fun normalizeChannelKey(value: String): String = value.trim() - .substringBefore('#') - .substringBefore('?') - .removeSuffix("/") - .replace("http://", "https://") - .replace( - Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), - "https://youtube.com", - ) - .withoutYoutubeTab() - -private fun String.withoutYoutubeTab(): String { - val uri = runCatching { URI(this) }.getOrNull() ?: return this - if (!uri.host.equals("youtube.com", ignoreCase = true)) return this - val segments = uri.path.split('/').filter(String::isNotBlank) - if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this - val path = "/${segments.dropLast(1).joinToString("/")}" - return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() -} - -private val YOUTUBE_CHANNEL_TABS = setOf( - "featured", - "videos", - "shorts", - "streams", - "playlists", - "community", - "about", -) diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt b/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt index aba44687..82079358 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt @@ -71,8 +71,3 @@ private fun toAllowedPlaylistItem(row: ResultRow): AllowedPlaylistItem = Allowed allowedAt = row[AllowedPlaylistsTable.allowedAt], global = row[AllowedPlaylistsTable.scope] == ALLOW_SCOPE_GLOBAL, ) - -internal fun normalizePlaylistKey(value: String): String = value.trim() - .substringBefore('#') - .removeSuffix("/") - .replace("http://", "https://") diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt b/src/main/kotlin/dev/typetype/server/services/BlockedService.kt index 3d91f801..d3c72031 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BlockedService.kt @@ -161,9 +161,6 @@ class BlockedService { } } -internal fun normalizeBlockedKeyword(value: String): String = - Normalizer.normalize(value, Normalizer.Form.NFKC).trim().lowercase() - internal fun containsBlockedKeyword(title: String, keyword: String): Boolean { val normalizedKeyword = normalizeBlockedKeyword(keyword) return normalizedKeyword.isNotEmpty() && normalizeBlockedKeyword(title).contains(normalizedKeyword) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt index d570645a..4979d8b9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt @@ -2,16 +2,16 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -class SubscriptionFeedCacheInvalidator( +class SubscriptionFeedCacheInvalidatorImpl( private val cache: CacheService, private val feedService: SubscriptionFeedService, -) { - suspend fun invalidate(userId: String) { +) : SubscriptionFeedCacheInvalidator { + override suspend fun invalidate(userId: String) { feedService.invalidate(userId) runCatching { cache.delete(SubscriptionFeedCacheKeys.shorts(userId)) } } - suspend fun awaitRefresh(userId: String) { + override suspend fun awaitRefresh(userId: String) { feedService.awaitRefresh(userId) } } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt index dc0b9ece..844b4e7c 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeTest.kt @@ -6,7 +6,7 @@ import dev.typetype.server.routes.subscriptionFeedRoutes import dev.typetype.server.services.AuthService import dev.typetype.server.services.PipePipeBackupImporterService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService import io.ktor.client.request.forms.MultiPartFormDataContent @@ -51,7 +51,7 @@ class SubscriptionFeedCacheInvalidationPipePipeTest { fun clean() = runBlocking { TestDatabase.truncateAll() cache.clear() - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt index 7e06f65d..9e83b903 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationPipePipeUnitTest.kt @@ -4,7 +4,7 @@ import dev.typetype.server.services.PipePipeBackupPersisterService import dev.typetype.server.services.PipePipeBackupSnapshotItem import dev.typetype.server.services.PipePipeBackupSubscriptionItem import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedCacheKeys import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService @@ -29,7 +29,7 @@ class SubscriptionFeedCacheInvalidationPipePipeUnitTest { TestDatabase.truncateAll() cache.clear() val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt index 7cabdca9..83c8bfc6 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheInvalidationTakeoutUnitTest.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.FavoritesService import dev.typetype.server.services.HistoryService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation -import dev.typetype.server.services.SubscriptionFeedCacheInvalidator +import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl import dev.typetype.server.services.SubscriptionFeedCacheKeys import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionsService @@ -36,7 +36,7 @@ class SubscriptionFeedCacheInvalidationTakeoutUnitTest { TestDatabase.truncateAll() cache.clear() val feed = SubscriptionFeedService(SubscriptionsService(), FakeChannelService(), cache) - SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidator(cache, feed)) + SubscriptionFeedCacheInvalidation.configure(SubscriptionFeedCacheInvalidatorImpl(cache, feed)) } @Test From 2ee58cdeabbf6257faa435421e8568ab63b1e54f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 21:21:22 +0200 Subject: [PATCH 037/137] refactor: move portability glue into server-portability module --- .../dev/typetype/server/portability/PortabilityEngineFactory.kt | 0 .../typetype/server/portability/TypeTypePortabilityAdapter.kt | 0 .../server/portability/TypeTypePortabilityCoreImport.kt | 0 .../typetype/server/portability/TypeTypePortabilityDataPort.kt | 0 .../typetype/server/portability/TypeTypePortabilityExport.kt | 0 .../server/portability/TypeTypePortabilityFilterImport.kt | 0 .../typetype/server/portability/TypeTypePortabilityImport.kt | 0 .../typetype/server/portability/TypeTypePortabilityReader.kt | 0 .../server/portability/TypeTypePortabilitySettingsImport.kt | 0 .../server/services/SubscriptionGroupBackupRepository.kt | 2 +- .../dev/typetype/server/services/TypeTypeBackupCategory.kt | 0 .../dev/typetype/server/services/TypeTypeBackupCoreRestore.kt | 0 .../dev/typetype/server/services/TypeTypeBackupFilterRestore.kt | 0 .../typetype/server/services/TypeTypeBackupLibraryRestore.kt | 0 .../dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt | 2 +- 15 files changed, 2 insertions(+), 2 deletions(-) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt (98%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt (100%) rename {src => server-portability/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt (98%) diff --git a/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngineFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapter.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityExport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityFilterImport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityImport.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityReader.kt diff --git a/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt rename to server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilitySettingsImport.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt index 3df60b25..4b1c5082 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupBackupRepository.kt @@ -12,7 +12,7 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import java.util.Locale import java.util.UUID -internal object SubscriptionGroupBackupRepository { +object SubscriptionGroupBackupRepository { suspend fun export( userId: String, subscriptionUrls: Set, diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCategory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupFilterRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupLibraryRestore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt rename to server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt index b8209cb9..18a89cfc 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupRestoreWriter.kt @@ -4,7 +4,7 @@ import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.models.TypeTypeBackupItem import dev.typetype.server.models.TypeTypeRestoreSummary -internal object TypeTypeBackupRestoreWriter { +object TypeTypeBackupRestoreWriter { suspend fun restore( userId: String, backup: TypeTypeBackupItem, From bce7098918a9e4a9bd452ed43bd00e2ad1d67aa6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 21:46:53 +0200 Subject: [PATCH 038/137] refactor: extract server-playback into Gradle module --- ....kt => TypetypeTokenSabrTokenClientTest.kt | 0 TypetypeYoutubeSessionPoTokenProviderTest.kt | 96 +++++ ...bePlayerClientFallbackStreamServiceTest.kt | 0 ...=> YoutubePlayerClientStreamServiceTest.kt | 0 build.gradle.kts | 1 + server-auth/build.gradle.kts | 6 + .../server/services/AuthSessionConfigTest.kt | 0 server-domain/build.gradle.kts | 3 + .../server/services/PublicCacheKey.kt | 2 +- .../services/RunCatchingNonCancellation.kt | 2 +- .../typetype/server/services/StreamService.kt | 0 .../server/services/YoutubeUrlDetector.kt | 2 +- server-playback/build.gradle.kts | 35 ++ .../services/ActiveSessionNowPlayingMapper.kt | 2 +- .../server/services/ActiveSessionRecord.kt | 2 +- .../server/services/ActiveSessionStrings.kt | 2 +- .../services/AuthenticatedSabrInfoCache.kt | 2 +- .../services/AuthenticatedSabrInfoService.kt | 6 +- .../services/AuthenticatedSabrPolicy.kt | 2 +- .../server/services/CachedSabrSegment.kt | 4 +- .../HomeRecommendationSessionIntent.kt | 0 .../HomeRecommendationSessionPersona.kt | 0 .../services/SabrAdaptiveInitialization.kt | 2 +- .../services/SabrBootstrapStreamService.kt | 4 +- .../services/SabrCachedSegmentLocator.kt | 4 +- .../services/SabrDashManifestBuilder.kt | 2 +- .../services/SabrDemandAttemptFinisher.kt | 2 +- .../server/services/SabrDemandDeadline.kt | 2 +- .../services/SabrDemandRecoveryAction.kt | 2 +- .../server/services/SabrDemandWatchdog.kt | 2 +- .../services/SabrDownloadFrameWriter.kt | 2 +- .../services/SabrDownloadInitialization.kt | 2 +- .../server/services/SabrDownloadRange.kt | 2 +- .../server/services/SabrDownloadStreamer.kt | 2 +- .../server/services/SabrFailureMessages.kt | 4 +- .../services/SabrFallbackStreamMapper.kt | 4 +- .../services/SabrFallbackStreamService.kt | 2 +- .../server/services/SabrHlsManifestBuilder.kt | 2 +- .../services/SabrInFlightDemandTracker.kt | 12 +- .../server/services/SabrInfoFetcher.kt | 2 +- .../server/services/SabrInfoRepository.kt | 2 +- .../server/services/SabrInfoSharedCache.kt | 2 +- .../server/services/SabrInitializationData.kt | 2 +- .../services/SabrInitializationPolicy.kt | 2 +- .../SabrInitializationSegmentFetcher.kt | 4 +- .../services/SabrLiveContinuationRequest.kt | 4 +- .../services/SabrLiveMediaNormalizer.kt | 4 +- .../server/services/SabrLivePlayback.kt | 12 +- .../services/SabrLivePlaybackDiscontinuity.kt | 4 +- .../server/services/SabrLivePumpStep.kt | 2 +- .../server/services/SabrLiveWarmupRequest.kt | 10 +- .../server/services/SabrManifestBuilder.kt | 2 +- .../server/services/SabrManifestTiming.kt | 2 +- .../server/services/SabrManifestUrlMapper.kt | 2 +- .../server/services/SabrMemorySegmentCache.kt | 2 +- .../server/services/SabrMimeAttributes.kt | 4 +- .../server/services/SabrPendingSeek.kt | 8 +- .../SabrPlaybackCachedSegmentLocator.kt | 6 +- .../services/SabrPlaybackDiagnostics.kt | 2 +- .../server/services/SabrPlaybackGeneration.kt | 2 +- .../services/SabrPlaybackInfoResolver.kt | 2 +- .../SabrPlaybackInitializationPreloader.kt | 4 +- .../services/SabrPlaybackManifestService.kt | 4 +- .../services/SabrPlaybackMediaFetcher.kt | 2 +- .../services/SabrPlaybackPreparation.kt | 2 +- .../services/SabrPlaybackSegmentResult.kt | 2 +- .../services/SabrPlaybackSegmentSelection.kt | 10 +- .../services/SabrPlaybackSessionService.kt | 2 +- .../server/services/SabrPlaybackStarter.kt | 4 +- .../server/services/SabrPlaybackState.kt | 2 +- .../server/services/SabrPlaybackStatus.kt | 2 +- .../server/services/SabrPlaybackWarmer.kt | 2 +- .../services/SabrPlayerContextRecovery.kt | 4 +- .../server/services/SabrPlayerInfoProbe.kt | 4 +- .../server/services/SabrPreparedInfoCache.kt | 2 +- .../services/SabrProtectedContextRecovery.kt | 2 +- .../services/SabrProtectedResponseGuard.kt | 4 +- .../server/services/SabrPumpConstants.kt | 0 .../server/services/SabrPumpCoordinator.kt | 10 +- .../server/services/SabrPumpLauncher.kt | 2 +- .../server/services/SabrPumpLogger.kt | 2 +- .../server/services/SabrPumpLoopPolicy.kt | 4 +- .../server/services/SabrPumpPolicy.kt | 2 +- .../server/services/SabrPumpRuntime.kt | 4 +- .../server/services/SabrSegmentCache.kt | 2 +- .../services/SabrSegmentDemandResolution.kt | 2 +- .../services/SabrSegmentDemandTracker.kt | 20 +- .../server/services/SabrSessionFactory.kt | 2 +- .../server/services/SabrSessionHolder.kt | 4 +- .../server/services/SabrSessionIdentity.kt | 2 +- .../server/services/SabrSessionKey.kt | 2 +- .../services/SabrSessionMediaFetcher.kt | 2 +- .../services/SabrSessionPlayerContext.kt | 2 +- .../server/services/SabrSessionProgress.kt | 16 +- .../server/services/SabrSessionPump.kt | 2 +- .../server/services/SabrSessionPumpLoop.kt | 2 +- .../server/services/SabrSessionPurpose.kt | 2 +- .../server/services/SabrSessionRegistry.kt | 2 +- .../server/services/SabrSessionResources.kt | 2 +- .../services/SabrSessionStorageTypes.kt | 2 +- .../server/services/SabrSessionStore.kt | 50 +-- .../services/SabrSessionStoreDefaults.kt | 2 +- .../services/SabrSessionTimeRequests.kt | 6 +- .../services/SabrSessionTokenGenerator.kt | 2 +- .../server/services/SabrStartupBootstrap.kt | 2 +- .../server/services/SabrTargetRequest.kt | 6 +- .../server/services/SabrTargetRequestShape.kt | 18 +- .../SabrUnauthorizedResponseRecovery.kt | 2 +- .../services/SabrWindowSegmentFetcher.kt | 2 +- .../TypetypeTokenSabrPoTokenProvider.kt | 2 +- .../server/services/YoutubeSessionService.kt | 0 .../services/YoutubeSessionTokenScope.kt | 0 .../AuthenticatedSabrInfoCacheTest.kt | 0 .../AuthenticatedSabrInfoServiceTest.kt | 0 .../server/services/PlaybackTestFixtures.kt | 77 ++++ .../SabrBootstrapStreamServiceTest.kt | 79 ++++ .../services/SabrCachedSegmentLocatorTest.kt | 0 .../services/SabrDashManifestBuilderTest.kt | 0 .../services/SabrDemandFailurePumpTest.kt | 0 .../services/SabrDemandWatchdogBackoffTest.kt | 0 .../SabrDemandWatchdogLifecycleTest.kt | 0 .../services/SabrDownloadFrameWriterTest.kt | 0 .../server/services/SabrDownloadRangeTest.kt | 0 .../services/SabrDownloadStreamerTest.kt | 0 .../services/SabrFallbackStreamServiceTest.kt | 0 .../SabrLiveContinuationRequestTest.kt | 27 ++ .../services/SabrLiveFutureRequestTest.kt | 0 .../services/SabrLiveMediaNormalizerTest.kt | 0 .../SabrLivePlaybackSessionServiceTest.kt | 0 .../server/services/SabrLivePlaybackTest.kt | 0 .../services/SabrLiveSessionWarmupTest.kt | 84 ++++ .../services/SabrMissingDemandRecoveryTest.kt | 0 .../SabrPlaybackCachedSegmentLocatorTest.kt | 0 .../services/SabrPlaybackDiagnosticsTest.kt | 0 .../services/SabrPlaybackInfoResolverTest.kt | 0 .../SabrPlaybackInitializationFailureTest.kt | 0 .../SabrPlaybackManifestServiceTest.kt | 122 ++++++ .../SabrPlaybackSessionIsolationTest.kt | 0 .../SabrPlaybackSessionServiceTest.kt | 0 .../services/SabrPlayerContextRecoveryTest.kt | 0 .../services/SabrPreparedInfoCacheTest.kt | 0 .../SabrProtectedResponseGuardTest.kt | 49 +++ .../services/SabrPumpCoordinatorTest.kt | 0 .../server/services/SabrPumpLauncherTest.kt | 0 .../server/services/SabrPumpPolicyTest.kt | 0 .../server/services/SabrPumpRuntimeTest.kt | 0 .../services/SabrSeekDiagnosticsTest.kt | 67 +++ .../services/SabrSeekRepositionPumpTest.kt | 0 .../server/services/SabrSegmentCacheTest.kt | 0 .../SabrSegmentDemandResolutionTest.kt | 0 .../services/SabrSegmentDemandTrackerTest.kt | 0 .../services/SabrSessionIdentityTest.kt | 0 .../server/services/SabrSessionKeyTest.kt | 0 .../services/SabrSessionPlayerContextTest.kt | 0 .../services/SabrSessionPumpLoopTest.kt | 0 .../server/services/SabrSessionPumpTest.kt | 0 .../services/SabrSessionRegistryTest.kt | 0 .../server/services/SabrSessionStoreTest.kt | 0 .../SabrTransientDemandFailureTest.kt | 189 ++++++++ .../SabrTransitioningLivePlaybackTest.kt | 0 .../SabrUnauthorizedResponseRecoveryTest.kt | 0 .../services/TestStreamResponseFixture.kt | 53 +++ .../TypetypeTokenSabrTokenClientTest.kt | 0 server-token-gateway/build.gradle.kts | 1 + .../server/services/NewPipeInitializer.kt | 0 .../TypetypeYoutubeSessionPoTokenProvider.kt | 2 +- settings.gradle.kts | 1 + .../services/DownloaderGatewayServiceTest.kt | 32 -- .../SabrAdaptiveInitializationTest.kt | 95 ++-- .../services/SabrFallbackStreamServiceTest.kt | 176 -------- .../services/SabrInitializationPolicyTest.kt | 173 +++++++- .../server/services/SabrLivePlaybackTest.kt | 300 ------------- .../services/SabrLiveProtocolProbeTest.kt | 404 ++++++++++++------ .../server/services/SabrLivePumpStepTest.kt | 208 ++++++--- .../SabrPlaybackInitializationFailureTest.kt | 204 --------- .../SabrPlaybackLiveGapServiceTest.kt | 246 +++++++---- .../services/SabrPreparedInfoCacheTest.kt | 229 ---------- .../server/services/SabrProbeConfig.kt | 266 ++++++++++-- .../server/services/SabrProbeDiagnostics.kt | 98 ++--- .../server/services/SabrProbeFetch.kt | 84 ++-- .../server/services/SabrProbeFetchResult.kt | 43 +- .../typetype/server/services/SabrProbeTest.kt | 99 +---- .../SabrProtectedContextRecoveryTest.kt | 116 +++-- .../server/services/SabrPumpRuntimeTest.kt | 139 ------ .../services/SabrRandomAccessProbeTest.kt | 266 ++++++------ .../SabrRecoverablePumpFailureTest.kt | 188 +++++--- .../server/services/SabrSessionStoreTest.kt | 78 ---- .../services/SabrSessionTimeRequestsTest.kt | 243 +++-------- .../StreamYouTubeSubtitleResolverTest.kt | 97 ----- ...petypeYoutubeSessionPoTokenProviderTest.kt | 139 +++--- 190 files changed, 2693 insertions(+), 2516 deletions(-) rename src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt => TypetypeTokenSabrTokenClientTest.kt (100%) create mode 100644 TypetypeYoutubeSessionPoTokenProviderTest.kt rename src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt => YoutubePlayerClientFallbackStreamServiceTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt => YoutubePlayerClientStreamServiceTest.kt (100%) rename {src => server-auth/src}/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt (100%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt (95%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt (72%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/StreamService.kt (100%) rename {src => server-domain/src}/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt (77%) create mode 100644 server-playback/build.gradle.kts rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt (81%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt (93%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt (100%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt (100%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt (94%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt (60%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt (60%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt (95%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt (84%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt (89%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt (92%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt (84%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt (90%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt (94%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt (81%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt (80%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt (94%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt (62%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt (73%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt (93%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt (88%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt (94%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt (82%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt (88%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt (91%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt (93%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt (100%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt (74%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt (95%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt (78%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt (95%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt (84%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt (98%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt (95%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt (90%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt (68%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt (76%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt (66%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt (99%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt (75%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt (59%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt (77%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt (93%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt (90%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt (87%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt (88%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt (93%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt (76%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt (96%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt (97%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt (94%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt (100%) rename {src => server-playback/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt (100%) rename {src => server-playback/src}/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt (100%) rename {src => server-playback/src}/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt rename src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt (100%) rename src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt (100%) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt rename src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt => server-playback/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt (100%) rename {src => server-token-gateway/src}/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt (94%) delete mode 100644 src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt delete mode 100644 src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt rename to TypetypeTokenSabrTokenClientTest.kt diff --git a/TypetypeYoutubeSessionPoTokenProviderTest.kt b/TypetypeYoutubeSessionPoTokenProviderTest.kt new file mode 100644 index 00000000..3a6a12e4 --- /dev/null +++ b/TypetypeYoutubeSessionPoTokenProviderTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken +import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider + +class TypetypeYoutubeSessionPoTokenProviderTest { + @AfterEach + fun clearAuthenticatedProvider(): Unit = + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) + + @Test + fun `exposes the session token only inside its scope`() { + TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { + assertEquals("visitor", currentToken()?.visitorData) + assertEquals("player-token", currentToken()?.poToken) + } + + assertNull(currentToken()) + } + + @Test + fun `restores the outer token after a nested scope`() { + TypetypeYoutubeSessionPoTokenProvider.withToken(token("outer", "outer-token")) { + TypetypeYoutubeSessionPoTokenProvider.withToken(token("inner", "inner-token")) { + assertEquals("inner", currentToken()?.visitorData) + } + assertEquals("outer", currentToken()?.visitorData) + } + + assertNull(currentToken()) + } + + @Test + fun `clears the token when the scoped call fails`() { + runCatching { + TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { + error("failed") + } + } + + assertNull(currentToken()) + } + + @Test + fun `uses the authenticated provider outside a SABR scope`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + assertEquals("auth", currentToken()?.visitorData) + assertEquals("auth-token", currentToken()?.poToken) + } + + @Test + fun `prefers the SABR token over the authenticated provider`() { + TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + + TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { + assertEquals("sabr", currentToken()?.visitorData) + assertEquals("sabr-token", currentToken()?.poToken) + } + } + + private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( + "MWEB", + "2.20260801.00.00", + "test-user-agent", + Localization("en", "US"), + ContentCountry("US"), + false, + ) + + private fun token(visitorData: String, playerToken: String) = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = playerToken, + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = visitorData, + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), + ) + + private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { + override fun getSessionPoToken( + clientName: String, + clientVersion: String, + userAgent: String?, + localization: Localization, + contentCountry: ContentCountry, + loggedIn: Boolean, + ) = YoutubeSessionPoToken(visitorData, poToken) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt b/YoutubePlayerClientFallbackStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt rename to YoutubePlayerClientFallbackStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt b/YoutubePlayerClientStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt rename to YoutubePlayerClientStreamServiceTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 6a81ea6f..3bd98cb5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { implementation(project(":server-downloader")) implementation(project(":server-domain")) implementation(project(":server-portability")) + implementation(project(":server-playback")) implementation(project(":server-sabr")) implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) diff --git a/server-auth/build.gradle.kts b/server-auth/build.gradle.kts index 58ed8c60..40d9fa0e 100644 --- a/server-auth/build.gradle.kts +++ b/server-auth/build.gradle.kts @@ -11,8 +11,14 @@ dependencies { implementation("com.password4j:password4j:1.8.4") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/services/AuthSessionConfigTest.kt diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts index 5a009688..24fddfdf 100644 --- a/server-domain/build.gradle.kts +++ b/server-domain/build.gradle.kts @@ -5,6 +5,9 @@ plugins { dependencies { api(project(":server-core")) + implementation(project(":server-downloader")) + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") implementation(project(":server-cache")) implementation(project(":server-db")) implementation("org.jetbrains.exposed:exposed-core:1.5.0") diff --git a/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt index 9b4fe439..75f1d981 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/PublicCacheKey.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.security.MessageDigest -internal object PublicCacheKey { +object PublicCacheKey { private val hex = "0123456789abcdef".toCharArray() fun of(area: String, vararg parts: String?): String = "$area:v2:${digest(parts)}" diff --git a/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt index f90a3e37..46d36da8 100644 --- a/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/RunCatchingNonCancellation.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException -internal inline fun runCatchingNonCancellation(block: () -> T): Result = try { +inline fun runCatchingNonCancellation(block: () -> T): Result = try { Result.success(block()) } catch (error: CancellationException) { throw error diff --git a/src/main/kotlin/dev/typetype/server/services/StreamService.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/StreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StreamService.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/StreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt rename to server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt index f086ba35..da830793 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/YoutubeUrlDetector.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.net.URI -internal fun isYoutubeUrl(url: String): Boolean = runCatching { +fun isYoutubeUrl(url: String): Boolean = runCatching { val host = URI(url).host?.lowercase() ?: return false host == "youtu.be" || host == "youtube.com" || host.endsWith(".youtube.com") }.getOrDefault(false) diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts new file mode 100644 index 00000000..ae42b1cc --- /dev/null +++ b/server-playback/build.gradle.kts @@ -0,0 +1,35 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-sabr")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-token-gateway")) + implementation(project(":server-downloader")) + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + implementation("org.slf4j:slf4j-api:2.0.16") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +kotlin { + jvmToolchain(25) +} + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt index da44b509..727c13fc 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionNowPlayingMapper.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.ActiveSessionNowPlayingItem import dev.typetype.server.models.SessionPlaybackProgressRequest import dev.typetype.server.models.SessionPlaybackStartRequest -internal object ActiveSessionNowPlayingMapper { +object ActiveSessionNowPlayingMapper { fun fromStart(request: SessionPlaybackStartRequest, now: Long): ActiveSessionNowPlayingItem = ActiveSessionNowPlayingItem( videoUrl = request.videoUrl.trim(), title = request.title.trim(), diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt index 7ebbe0c2..4abe3612 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionRecord.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ActiveSessionNowPlayingItem -internal data class ActiveSessionRecord( +data class ActiveSessionRecord( val id: String, val userId: String, val username: String?, diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt index eb52953a..5d9b5d8d 100644 --- a/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/ActiveSessionStrings.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object ActiveSessionStrings { +object ActiveSessionStrings { fun text(value: String?): String? = value?.trim()?.take(MAX_TEXT_LENGTH)?.takeIf { it.isNotEmpty() } fun userAgent(value: String?): String? = value?.trim()?.take(MAX_USER_AGENT_LENGTH)?.takeIf { it.isNotEmpty() } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt index 1772414c..1556f569 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCache.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.withTimeout import java.time.Duration import java.util.concurrent.ConcurrentHashMap -internal class AuthenticatedSabrInfoCache( +class AuthenticatedSabrInfoCache( ttl: Duration = Duration.ofMinutes(5), maxEntries: Int = 256, private val timeoutMs: Long = AuthenticatedSabrPolicy.INFO_TIMEOUT_MS, diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt index 1768eaec..40dd2abf 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoService.kt @@ -13,7 +13,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.SabrAdapter import org.slf4j.LoggerFactory -internal class AuthenticatedSabrInfoService( +class AuthenticatedSabrInfoService( private val youtubeSessionService: YoutubeSessionService, private val tokenClient: TypetypeTokenSabrTokenClient, private val visitorDataFetcher: () -> String = AuthenticatedYoutubeVisitorData::fetch, @@ -71,14 +71,14 @@ internal class AuthenticatedSabrInfoService( } } -internal sealed interface AuthenticatedSabrInfoResult { +sealed interface AuthenticatedSabrInfoResult { data object NotConnected : AuthenticatedSabrInfoResult data object Failed : AuthenticatedSabrInfoResult data object TimedOut : AuthenticatedSabrInfoResult data class Ready(val prepared: SabrPreparedInfo) : AuthenticatedSabrInfoResult } -internal fun interface AuthenticatedSabrProbe { +fun interface AuthenticatedSabrProbe { fun fetch(videoId: String, token: YoutubeSessionPoToken): YoutubeSabrInfo } diff --git a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt index a0a67f70..d67a3ea7 100644 --- a/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/AuthenticatedSabrPolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object AuthenticatedSabrPolicy { +object AuthenticatedSabrPolicy { const val INFO_TIMEOUT_MS = 15_000L const val STREAM_TIMEOUT_MS = 20_000L const val TIMEOUT_CODE = "authenticated_sabr_timeout" diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt index 443c6b6d..3a5c7602 100644 --- a/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/CachedSabrSegment.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import java.util.Base64 -internal class CachedSabrSegment( +class CachedSabrSegment( val itag: Int, val sequence: Int, val init: Boolean, @@ -36,7 +36,7 @@ internal class CachedSabrSegment( val length: Int get() = bytes.size } -internal fun SabrMediaSegment.toCachedSabrSegment( +fun SabrMediaSegment.toCachedSabrSegment( mimeType: String, bytes: ByteArray = data, ): CachedSabrSegment = CachedSabrSegment( diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntent.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionPersona.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt index 70b2e088..cf5afd08 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrAdaptiveInitialization.kt @@ -6,7 +6,7 @@ import kotlinx.coroutines.runInterruptible import org.schabi.newpipe.extractor.localization.Localization import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrAdaptiveInitialization { +object SabrAdaptiveInitialization { private val localization = Localization("en", "US") suspend fun fetch( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt index 771aed8c..a50ec0c5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -internal class SabrBootstrapStreamService( +class SabrBootstrapStreamService( private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, ) : StreamService { @@ -20,5 +20,5 @@ internal class SabrBootstrapStreamService( } } -internal fun youtubeVideoId(url: String): String? = +fun youtubeVideoId(url: String): String? = Regex("(?:[?&]v=|/shorts/|youtu\\.be/)([A-Za-z0-9_-]{6,})").find(url)?.groupValues?.get(1) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt index a532cdfd..8b1baa41 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrCachedSegmentLocator.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrSession -internal fun YoutubeSabrSession.findCachedMediaAt( +fun YoutubeSabrSession.findCachedMediaAt( format: YoutubeSabrFormat, targetMs: Long, predictedSequence: Int, @@ -53,7 +53,7 @@ private fun SabrMediaSegment.startsWithinFollowingRange( return header.startMs >= 0L && leadMs in -TIMING_TOLERANCE_MS..maximumLeadMs } -internal fun maximumFollowingLeadMs(durationMs: Long, sequenceDistance: Int): Long { +fun maximumFollowingLeadMs(durationMs: Long, sequenceDistance: Int): Long { val distance = sequenceDistance.coerceAtLeast(1).toLong() val toleranceMs = LIVE_FOLLOWING_TIMING_TOLERANCE_MS val boundedDurationMs = durationMs.coerceAtMost((Long.MAX_VALUE - toleranceMs) / distance) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt index 6c443d75..62613783 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDashManifestBuilder.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrDashManifestBuilder { +object SabrDashManifestBuilder { fun build( videoId: String, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index 2727597c..14693b63 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession -internal object SabrDemandAttemptFinisher { +object SabrDemandAttemptFinisher { fun interruptCompletedInFlightDemand(holder: SabrSessionHolder, demand: SabrInFlightDemand): Boolean = synchronized(holder) { val state = holder.playbackState() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt index 39aa90b6..de717f30 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrDemandDeadline( +class SabrDemandDeadline( private val timeoutMs: Long, private val maximumBackoffExtensionMs: Long = SabrPumpPolicy.MAX_DEMAND_BACKOFF_EXTENSION_MS, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt index c7525c26..f1834a49 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandRecoveryAction.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrDemandRecoveryAction { +enum class SabrDemandRecoveryAction { WAIT, READVERTISE_TRACK, } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index 2a0e3874..faa6bdb2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import kotlinx.coroutines.delay -internal class SabrDemandWatchdog( +class SabrDemandWatchdog( private val clock: () -> Long = System::currentTimeMillis, private val intervalMs: Long = SabrPumpPolicy.IDLE_POLL_MS, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt index 5d2eeee6..80809b00 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadFrameWriter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.io.InputStream import java.io.OutputStream -internal class SabrDownloadFrameWriter(private val output: OutputStream) { +class SabrDownloadFrameWriter(private val output: OutputStream) { private val header = ByteArray(FRAME_HEADER_SIZE) private val copyBuffer = ByteArray(COPY_BUFFER_SIZE) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt index f4ed0722..069aa5c2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadInitialization.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrDownloadInitialization { +object SabrDownloadInitialization { suspend fun fetch( store: SabrSessionStore, holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt index f5f80626..431e0ceb 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadRange.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal data class SabrDownloadRange( +data class SabrDownloadRange( val part: Int = 0, val parts: Int = 1, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt index 16fdcc33..538eb082 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDownloadStreamer.kt @@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory import java.io.IOException import java.io.OutputStream -internal class SabrDownloadStreamer( +class SabrDownloadStreamer( private val store: SabrSessionStore, private val pumpTimeoutMs: Long = PUMP_TIMEOUT_MS, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt index 6a1b30c9..10557228 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFailureMessages.kt @@ -1,8 +1,8 @@ package dev.typetype.server.services -internal const val SABR_RECOVERABLE_FAILURE_PREFIX = "SABR recoverable failure:" +const val SABR_RECOVERABLE_FAILURE_PREFIX = "SABR recoverable failure:" -internal fun sabrRecoverableFailureMessage(message: String?): String { +fun sabrRecoverableFailureMessage(message: String?): String { val failure = message ?: "Unknown SABR recoverable failure" if (failure.contains("SABR spool", ignoreCase = true)) return failure return "$SABR_RECOVERABLE_FAILURE_PREFIX $failure" diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt index 305edca5..38b84e5d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamMapper.kt @@ -8,7 +8,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrInfo): StreamResponse { +fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrInfo): StreamResponse { val video = info.formats.filter { it.isVideo }.mapNotNull { it.toFallbackVideo(videoId) } val audio = info.formats.filter { it.isAudio }.mapNotNull { it.toFallbackAudio(videoId) } return copy( @@ -19,7 +19,7 @@ internal fun StreamResponse.withSabrFallback(videoId: String, info: YoutubeSabrI ) } -internal fun TokenYoutubeSession.toFallbackStreamResponse(videoId: String): StreamResponse = StreamResponse( +fun TokenYoutubeSession.toFallbackStreamResponse(videoId: String): StreamResponse = StreamResponse( id = videoId, title = title, uploaderName = author, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt index 5be6711a..342b2af7 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.StreamResponse import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -internal class SabrFallbackStreamService( +class SabrFallbackStreamService( private val delegate: StreamService, private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt index 2ef01b4d..069b34d6 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrHlsManifestBuilder.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState import java.util.Locale -internal object SabrHlsManifestBuilder { +object SabrHlsManifestBuilder { fun buildMaster( videoId: String, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt index e0f6ae20..0b73deed 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap -internal class SabrInFlightDemand( +class SabrInFlightDemand( val request: SabrSegmentRequest, val identity: String, val registeredAtMs: Long, @@ -21,7 +21,7 @@ internal class SabrInFlightDemand( } } -internal object SabrInFlightDemandTracker { +object SabrInFlightDemandTracker { private val demands = ConcurrentHashMap() fun begin( @@ -55,17 +55,17 @@ internal object SabrInFlightDemandTracker { fun clearAll(): Unit = demands.clear() } -internal fun SabrSessionHolder.beginInFlightSegmentDemand( +fun SabrSessionHolder.beginInFlightSegmentDemand( request: SabrSegmentRequest, identity: String, futureLiveRequest: Boolean, ): Boolean = SabrInFlightDemandTracker.begin(this, request, identity, futureLiveRequest) -internal fun SabrSessionHolder.inFlightSegmentDemand(): SabrInFlightDemand? = +fun SabrSessionHolder.inFlightSegmentDemand(): SabrInFlightDemand? = SabrInFlightDemandTracker.current(this) -internal fun SabrSessionHolder.finishInFlightSegmentDemand(identity: String): Boolean = +fun SabrSessionHolder.finishInFlightSegmentDemand(identity: String): Boolean = SabrInFlightDemandTracker.finish(this, identity) -internal fun SabrSessionHolder.clearInFlightSegmentDemand(): Unit = +fun SabrSessionHolder.clearInFlightSegmentDemand(): Unit = SabrInFlightDemandTracker.clear(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index a1e9b15c..c81b22fc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -9,7 +9,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import org.slf4j.LoggerFactory -internal class SabrInfoFetcher( +class SabrInfoFetcher( private val tokenClient: TypetypeTokenSabrTokenClient, private val sessionClient: TypetypeTokenYoutubeSessionClient? = null, private val infoCache: SabrPreparedInfoCache = SabrPreparedInfoCache(), diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt index f0fbd251..a84b0f74 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoRepository.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration -internal class SabrInfoRepository( +class SabrInfoRepository( infoCache: SabrPreparedInfoCache, sharedCache: CacheService?, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt index eabccc84..d0b61d4f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoSharedCache.kt @@ -8,7 +8,7 @@ import java.io.ObjectInputStream import java.io.ObjectOutputStream import java.util.Base64 -internal class SabrInfoSharedCache(private val cache: CacheService?) { +class SabrInfoSharedCache(private val cache: CacheService?) { suspend fun getPlayback(videoId: String): YoutubeSabrInfo? = get("sabr:info:v2:$videoId") suspend fun getInitialization(videoId: String): YoutubeSabrInfo? = get("sabr:init-info:v1:$videoId") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt index 0b9788ca..d4511d47 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationData.kt @@ -11,7 +11,7 @@ import java.util.Base64 import java.util.Collections import java.util.WeakHashMap -internal object SabrInitializationData { +object SabrInitializationData { private const val CACHE_TTL_SECONDS = 21_600L private val memoryCache = BoundedExpiringCache( maxEntries = 512, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt index 6f85a8a0..5c5062a3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationPolicy.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal object SabrInitializationPolicy { +object SabrInitializationPolicy { fun warmFormats( audioOnly: Boolean, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt index 1ccab335..3c4b7395 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInitializationSegmentFetcher.kt @@ -24,7 +24,7 @@ internal suspend fun fetchSabrInitializationSegment( holder.withPlayerContext { fetchSegment(request, localization) } } result.onFailure { error -> - logger.warn( + initFetcherLogger.warn( "sabr_init event=fetch_failed videoId={} itag={} errorType={} error={}", holder.key.videoId, request.format.itag, @@ -41,4 +41,4 @@ internal suspend fun fetchSabrInitializationSegment( } } -private val logger = LoggerFactory.getLogger("SabrInitializationSegmentFetcher") +private val initFetcherLogger = LoggerFactory.getLogger("SabrInitializationSegmentFetcher") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt index d89e3736..d522aa9c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveContinuationRequest.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrBufferedRange import dev.typetype.server.sabr.YoutubeSabrFormat -internal inline fun withLiveContinuationRequestShape( +inline fun withLiveContinuationRequestShape( holder: SabrSessionHolder, block: () -> T, ): T { @@ -24,7 +24,7 @@ internal inline fun withLiveContinuationRequestShape( } } -private fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { +@PublishedApi internal fun SabrSessionHolder.continuationRange(format: YoutubeSabrFormat): SabrBufferedRange? { observedMediaSegment(format) ?: return null val sequence = lastServedSequence(format) ?: (playbackStartSequence(format, requestedSeekTimeMs() ?: playerTimeMs()) - 1).coerceAtLeast(0) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt index 580dcfba..75380261 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizer.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services -internal data class SabrLiveMediaParts( +data class SabrLiveMediaParts( val initialization: ByteArray, val media: ByteArray, ) -internal object SabrLiveMediaNormalizer { +object SabrLiveMediaNormalizer { fun split(mimeType: String, data: ByteArray): SabrLiveMediaParts? = when { mimeType.substringBefore(';').trim().lowercase().endsWith("/mp4") -> splitMp4(data) mimeType.substringBefore(';').trim().lowercase().endsWith("/webm") -> splitWebM(data) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index 0507aa72..517085e3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal data class SabrLivePlaybackSnapshot( +data class SabrLivePlaybackSnapshot( val active: Boolean, val postLiveDvr: Boolean, val headSequence: Long, @@ -14,7 +14,7 @@ internal data class SabrLivePlaybackSnapshot( val targetLatencyMs: Long, ) -internal fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? { +fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? { val state = session.streamState val postLiveDvr = runCatching { state.isPostLiveDvr }.getOrDefault(false) val sessionLive = runCatching { session.isLive }.getOrDefault(false) @@ -57,7 +57,7 @@ internal fun SabrSessionHolder.livePlaybackSnapshot(): SabrLivePlaybackSnapshot? ) } -internal fun SabrSessionHolder.resolvePlaybackStartMs(requestedStartMs: Long): Long { +fun SabrSessionHolder.resolvePlaybackStartMs(requestedStartMs: Long): Long { val requested = requestedStartMs.coerceAtLeast(0L) val live = livePlaybackSnapshot() ?: return requested if (!live.active) return requested.coerceAtMost(live.seekableEndMs.takeIf { it > 0L } ?: requested) @@ -73,7 +73,7 @@ private fun SabrSessionHolder.availableLiveMediaStartMs(): Long? { return maxOf(audioStartMs, videoStartMs) } -internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false if (session.getCachedSegment(request) != null) return false @@ -105,7 +105,7 @@ internal fun SabrSessionHolder.isFutureLiveRequest(request: SabrSegmentRequest): return requestStartMs > 0L && requestStartMs >= completeEndMs && !isHistoricalLiveRequest(request) } -internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false val observed = observedMediaSegment(request.format) ?: return false @@ -117,7 +117,7 @@ internal fun SabrSessionHolder.isHistoricalLiveRequest(request: SabrSegmentReque return request.sequenceNumber < observed.header.sequenceNumber } -internal fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = +fun SabrSessionHolder.liveRetryAfterMs(blockedRequests: List = emptyList()): Long = DEFAULT_PLAYBACK_RETRY_MS private fun dev.typetype.server.sabr.YoutubeSabrStreamState.observedEndMs( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt index 91ed693c..84140e8a 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlaybackDiscontinuity.kt @@ -3,14 +3,14 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.isLiveDemandOutsideRecoverableWindow(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.isLiveDemandOutsideRecoverableWindow(request: SabrSegmentRequest): Boolean { if (request.isInitializationSegment) return false val live = livePlaybackSnapshot()?.takeIf { it.active } ?: return false val requestEndMs = playbackSegmentEndMs(request.format, request.sequenceNumber) return requestEndMs > 0L && live.headTimeMs - requestEndMs > MAX_RECOVERABLE_LIVE_GAP_MS } -internal fun SabrSessionHolder.failLivePlaybackDiscontinuity( +fun SabrSessionHolder.failLivePlaybackDiscontinuity( format: YoutubeSabrFormat, targetMs: Long, segment: CachedSabrSegment, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt index ac33f9f7..cbd6e9e9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt @@ -22,7 +22,7 @@ internal suspend fun pumpLiveReadAhead( return cached > 0 && holder.observedLiveAheadMs() < targetAheadMs } -internal fun SabrSessionHolder.observedLiveAheadMs(): Long { +fun SabrSessionHolder.observedLiveAheadMs(): Long { val audioEndMs = observedMediaEndMs(audioFormat) ?: return 0L val videoEndMs = if (isVideoActive()) observedMediaEndMs(videoFormat) ?: return 0L else Long.MAX_VALUE return (minOf(audioEndMs, videoEndMs) - playerTimeMs()).coerceAtLeast(0L) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt index 189488ab..e72039ee 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLiveWarmupRequest.kt @@ -4,13 +4,13 @@ import dev.typetype.server.sabr.SabrBufferedRange import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -internal data class SabrLiveWarmupTarget( +data class SabrLiveWarmupTarget( val sequence: Int, val timeMs: Long, val segmentDurationMs: Long, ) -internal fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { +fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { val live = livePlaybackSnapshot() ?.takeIf { it.active && it.headSequence > 1L && it.headTimeMs > 0L } ?: return null @@ -27,7 +27,7 @@ internal fun SabrSessionHolder.liveWarmupTarget(): SabrLiveWarmupTarget? { return SabrLiveWarmupTarget(targetSequence, targetTimeMs, segmentDurationMs) } -internal fun SabrSessionHolder.advanceLiveWarmupTarget( +fun SabrSessionHolder.advanceLiveWarmupTarget( target: SabrLiveWarmupTarget, segments: List, ): SabrLiveWarmupTarget { @@ -52,7 +52,7 @@ internal fun SabrSessionHolder.advanceLiveWarmupTarget( ) } -internal inline fun withLiveWarmupRequestShape( +inline fun withLiveWarmupRequestShape( holder: SabrSessionHolder, target: SabrLiveWarmupTarget?, block: () -> T, @@ -74,7 +74,7 @@ internal inline fun withLiveWarmupRequestShape( } } -private fun YoutubeSabrFormat.liveWarmupRange(targetSequence: Int, targetTimeMs: Long): SabrBufferedRange { +@PublishedApi internal fun YoutubeSabrFormat.liveWarmupRange(targetSequence: Int, targetTimeMs: Long): SabrBufferedRange { val bufferedSequence = (targetSequence - 1).coerceAtLeast(0) return SabrBufferedRange( itag, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt index 33b45d76..16cb10e5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestBuilder.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrManifestBuilder { +object SabrManifestBuilder { fun build( videoId: String, audio: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt index 433b3aec..5d28f032 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestTiming.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState -internal object SabrManifestTiming { +object SabrManifestTiming { fun videoDurationSec( audio: YoutubeSabrFormat, video: YoutubeSabrFormat, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt index 101ea2a6..e100171b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrManifestUrlMapper.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem -internal fun StreamResponse.withSabrManifestUrls(): StreamResponse = copy( +fun StreamResponse.withSabrManifestUrls(): StreamResponse = copy( videoStreams = videoStreams.map { it.withSabrManifestUrl(id) }, videoOnlyStreams = videoOnlyStreams.map { it.withSabrManifestUrl(id) }, audioStreams = audioStreams.map { it.withSabrManifestUrl(id) }, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt index e0c827de..ac4d01fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMemorySegmentCache.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrMemorySegmentCache(private val maxBytes: Long) { +class SabrMemorySegmentCache(private val maxBytes: Long) { private val segments = LinkedHashMap(64, 0.75f, true) private var bytes = 0L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt similarity index 81% rename from src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt index 4b99aaca..39346c8c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrMimeAttributes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun splitMime(mime: String): Pair { +fun splitMime(mime: String): Pair { val parts = mime.split(";", limit = 2) val container = parts[0].trim() val codecs = if (parts.size > 1) { @@ -13,7 +13,7 @@ internal fun splitMime(mime: String): Pair { return container to codecs } -internal fun videoSizeAttr(video: YoutubeSabrFormat): String { +fun videoSizeAttr(video: YoutubeSabrFormat): String { val width = video.width.takeIf { it > 0 } val height = video.height.takeIf { it > 0 } return if (width != null && height != null) " width=\"$width\" height=\"$height\"" else "" diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt index 43170eca..1e478709 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPendingSeek.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest -internal fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): Boolean { +fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): Boolean { pendingRefetchRequest()?.takeIf { it.matches(request) }?.let { consumeRefetch() setPlaybackState(SabrPlaybackState.REPOSITIONING) @@ -18,13 +18,13 @@ internal fun SabrSessionHolder.consumeMatchingSeek(request: SabrSegmentRequest): return false } -internal fun SabrSessionHolder.prepareForExplicitRewind(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.prepareForExplicitRewind(request: SabrSegmentRequest): Unit = session.prepareForRewind(request, explicitSeekPositionMs()) -internal fun SabrSessionHolder.prepareForExplicitForwardJump(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.prepareForExplicitForwardJump(request: SabrSegmentRequest): Unit = session.prepareForForwardJump(request, explicitSeekPositionMs()) -internal fun SabrSessionHolder.prepareForHistoricalLiveRewind(request: SabrSegmentRequest): Unit { +fun SabrSessionHolder.prepareForHistoricalLiveRewind(request: SabrSegmentRequest): Unit { val seekPositionMs = requestedSeekTimeMs()?.takeIf { request.contains(this, it) } if (seekPositionMs == null) session.prepareForRewind(request) else session.prepareForRewind(request, seekPositionMs) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt index 1eee1935..c72c0837 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocator.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal suspend fun SabrSessionStore.findCachedPlaybackMediaAt( +suspend fun SabrSessionStore.findCachedPlaybackMediaAt( holder: SabrSessionHolder, format: YoutubeSabrFormat, targetMs: Long, @@ -41,7 +41,7 @@ private suspend fun SabrSessionStore.cachedMedia( return cachedSegment(holder, SabrSegmentRequest.media(format, sequence)) } -internal fun CachedSabrSegment.coversPlaybackTime( +fun CachedSabrSegment.coversPlaybackTime( holder: SabrSessionHolder, format: YoutubeSabrFormat, targetMs: Long, @@ -51,7 +51,7 @@ internal fun CachedSabrSegment.coversPlaybackTime( return targetMs >= effectiveStartMs - TIMING_TOLERANCE_MS && targetMs < effectiveStartMs + effectiveDurationMs } -internal fun CachedSabrSegment.isAcceptableLiveFollowingSegment( +fun CachedSabrSegment.isAcceptableLiveFollowingSegment( expectedSequence: Int, targetMs: Long, continuesServedTrack: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt index 2c003d40..3001f288 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackDiagnostics.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap -internal object SabrPlaybackDiagnostics { +object SabrPlaybackDiagnostics { private val blockers = ConcurrentHashMap() fun record(holder: SabrSessionHolder, request: SabrSegmentRequest, message: String?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt similarity index 62% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt index ddbd5804..a0d66257 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackGeneration.kt @@ -1,4 +1,4 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.nextReplacementGeneration(): Long = +fun SabrSessionHolder.nextReplacementGeneration(): Long = activeGeneration().let { if (it == Long.MAX_VALUE) it else it + 1L } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt index 0df44214..b9d2f033 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolver.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrPlaybackInfoResolver( +class SabrPlaybackInfoResolver( private val sessionStore: SabrSessionStore, private val authenticatedInfoService: AuthenticatedSabrInfoService?, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt index 10064135..89208786 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withTimeoutOrNull -internal data class SabrPlaybackInitializationPreloadResult( +data class SabrPlaybackInitializationPreloadResult( val video: ByteArray?, val audio: ByteArray?, ) { @@ -18,7 +18,7 @@ internal data class SabrPlaybackInitializationPreloadResult( }.joinToString() } -internal object SabrPlaybackInitializationPreloader { +object SabrPlaybackInitializationPreloader { suspend fun preload( sessionStore: SabrSessionStore, holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt index c713e7a0..61329705 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackManifestService.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrPlaybackManifestService { +class SabrPlaybackManifestService { fun build(holder: SabrSessionHolder, mediaBasePath: String): SabrPlaybackManifestResult { val state = holder.session.streamState val knownAudio = maxOf(state.getEndSegment(holder.audioFormat), state.getMaxSegment(holder.audioFormat).toLong()) @@ -53,7 +53,7 @@ internal class SabrPlaybackManifestService { } } -internal sealed class SabrPlaybackManifestResult { +sealed class SabrPlaybackManifestResult { data class Ready(val manifest: String) : SabrPlaybackManifestResult() data class Retry(val status: String) : SabrPlaybackManifestResult() } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt index bb1bd6c9..86e1b330 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackMediaFetcher.kt @@ -7,7 +7,7 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { +class SabrPlaybackMediaFetcher(private val sessionStore: SabrSessionStore) { suspend fun fetch( holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt index edc38946..05156fc0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackPreparation.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal data class SabrPlaybackPreparation( +data class SabrPlaybackPreparation( val holder: SabrSessionHolder, val startTimeMs: Long, val ready: Boolean, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt index 3f2cfc7c..e994b04e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentResult.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment -internal sealed class SabrPlaybackSegmentResult { +sealed class SabrPlaybackSegmentResult { data class Ready(val mimeType: String, val bytes: ByteArray) : SabrPlaybackSegmentResult() data class Stream( val mimeType: String, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt index 93bc0fd8..35c2e614 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSegmentSelection.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { +fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, playerTimeMs: Long): Int { liveSequenceAt(format, playerTimeMs)?.let { return it } val mappedSequence = session.streamState .getSegmentNumberAtOrAfterTimeMs(format, playerTimeMs.coerceAtLeast(0L)) @@ -11,7 +11,7 @@ internal fun SabrSessionHolder.playbackStartSequence(format: YoutubeSabrFormat, return mappedSequence.coerceAtMost(endSequence.takeIf { it > 0 } ?: mappedSequence) } -internal fun SabrSessionHolder.playbackContinuationSequence( +fun SabrSessionHolder.playbackContinuationSequence( format: YoutubeSabrFormat, playerTimeMs: Long, continueAfterLastServed: Boolean, @@ -20,7 +20,7 @@ internal fun SabrSessionHolder.playbackContinuationSequence( ?.plus(1) ?: playbackStartSequence(format, playerTimeMs) -internal fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, sequence: Int): Long { +fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, sequence: Int): Long { val observed = observedMediaSegment(format) val observedStartMs = observed?.header?.startMs?.takeIf { it >= 0L } if (observed != null && observedStartMs != null) { @@ -31,7 +31,7 @@ internal fun SabrSessionHolder.playbackSegmentStartMs(format: YoutubeSabrFormat, return session.streamState.getSegmentStartMs(format, sequence).coerceAtLeast(0L) } -internal fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrFormat, sequence: Int): Long { +fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrFormat, sequence: Int): Long { val observed = observedMediaSegment(format) val observedStartMs = observed?.header?.startMs?.takeIf { it >= 0L } val observedDurationMs = observed?.header?.durationMs?.takeIf { it > 0L } @@ -47,7 +47,7 @@ internal fun SabrSessionHolder.playbackSegmentDurationMs(format: YoutubeSabrForm return (session.streamState.getSegmentEndMs(format, sequence) - startMs).coerceAtLeast(1L) } -internal fun SabrSessionHolder.playbackSegmentEndMs(format: YoutubeSabrFormat, sequence: Int): Long = +fun SabrSessionHolder.playbackSegmentEndMs(format: YoutubeSabrFormat, sequence: Int): Long = playbackSegmentStartMs(format, sequence) + playbackSegmentDurationMs(format, sequence) private fun SabrSessionHolder.liveSequenceAt(format: YoutubeSabrFormat, playerTimeMs: Long): Int? { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index b19fc4dd..93e5f21e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { +class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { private val mediaFetcher = SabrPlaybackMediaFetcher(sessionStore) suspend fun prepare( videoId: String, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt index e2121ed0..644e25a0 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStarter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrPlaybackStarter { +object SabrPlaybackStarter { fun start( store: SabrSessionStore, holder: SabrSessionHolder, @@ -33,7 +33,7 @@ internal object SabrPlaybackStarter { private val logger = LoggerFactory.getLogger(SabrPlaybackStarter::class.java) } -internal fun SabrSessionHolder.requestPlaybackReposition(playerTimeMs: Long, generation: Long): Unit { +fun SabrSessionHolder.requestPlaybackReposition(playerTimeMs: Long, generation: Long): Unit { val targetFormat = if (isVideoActive()) videoFormat else audioFormat val request = SabrSegmentRequest.media(targetFormat, playbackStartSequence(targetFormat, playerTimeMs)) val companion = audioFormat.takeIf { isVideoActive() }?.let { format -> diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt similarity index 82% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt index 3739acba..8e7c3d67 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackState.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrPlaybackState { +enum class SabrPlaybackState { IDLE, PREPARING, REQUESTING, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt index 4809cf53..967c12a1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackStatus.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.util.concurrent.atomic.AtomicReference -internal class SabrPlaybackStatus { +class SabrPlaybackStatus { private val state = AtomicReference(SabrPlaybackState.IDLE) private val terminalError = AtomicReference() private val networkError = AtomicReference() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt index 45c37f49..cd10cf32 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackWarmer.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import org.slf4j.LoggerFactory -internal class SabrPlaybackWarmer { +class SabrPlaybackWarmer { suspend fun preflight(store: SabrSessionStore, holder: SabrSessionHolder, playerTimeMs: Long): Boolean { holder.setActiveTracks(videoActive = true, audioActive = true) store.ensureWarmed(holder) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt index f28cfbbc..506d5f22 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerContextRecovery.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrProtocolException import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo -internal class SabrPlayerContextRecovery( +class SabrPlayerContextRecovery( private val videoId: String, initialToken: SabrTokenBundle, private val tokenClient: TypetypeTokenSabrTokenClient, @@ -46,7 +46,7 @@ internal class SabrPlayerContextRecovery( this is SabrProtocolException && message == "Player response has no streamingData for $profile" } -internal sealed interface SabrPlayerProbeResult { +sealed interface SabrPlayerProbeResult { data class Success( val info: YoutubeSabrInfo, val token: SabrTokenBundle, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt index c5ba312b..5874ae01 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlayerInfoProbe.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.SabrAdapter -internal fun interface SabrPlayerInfoProbe { +fun interface SabrPlayerInfoProbe { fun fetch( videoId: String, profile: YoutubeSabrClientProfile, @@ -14,7 +14,7 @@ internal fun interface SabrPlayerInfoProbe { ): YoutubeSabrInfo } -internal object PipePipeSabrPlayerInfoProbe : SabrPlayerInfoProbe { +object PipePipeSabrPlayerInfoProbe : SabrPlayerInfoProbe { private val localization = Localization("en", "US") private val contentCountry = ContentCountry("US") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt index b7f160ff..336628d3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPreparedInfoCache.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.time.Duration -internal class SabrPreparedInfoCache( +class SabrPreparedInfoCache( private val ttl: Duration = Duration.ofMinutes(10), maxEntries: Int = 256, clock: () -> Long = System::currentTimeMillis, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt index 47058543..c5cea9ef 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedContextRecovery.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrProtectedContextRecovery( +class SabrProtectedContextRecovery( private val tokenClient: TypetypeTokenSabrTokenClient, ) { fun refreshIfRejected(videoId: String, rejectedVisitorData: String?) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt index 05d7f326..63f6f0b5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrProtectedResponseGuard.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class SabrProtectedResponseGuard( +class SabrProtectedResponseGuard( private val maximumResponses: Int = SabrPumpPolicy.MAX_PROTECTED_NO_MEDIA_RESPONSES, ) { private var lastResponseNumber: Int? = null @@ -42,4 +42,4 @@ internal class SabrProtectedResponseGuard( } } -internal class SabrProtectedNoMediaException(message: String) : RuntimeException(message) +class SabrProtectedNoMediaException(message: String) : RuntimeException(message) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpConstants.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt index 0344333d..95695438 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpCoordinator.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.withTimeoutOrNull import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong -internal class SabrPumpCoordinator { +class SabrPumpCoordinator { private val started = AtomicBoolean(false) private val wakeVersion = AtomicLong() private val wakeups = Channel(Channel.CONFLATED) @@ -30,13 +30,13 @@ internal class SabrPumpCoordinator { } } -internal fun SabrSessionHolder.markPumpStarted(): Boolean = pumpCoordinator.markStarted() +fun SabrSessionHolder.markPumpStarted(): Boolean = pumpCoordinator.markStarted() -internal fun SabrSessionHolder.markPumpStopped(): Unit = pumpCoordinator.markStopped() +fun SabrSessionHolder.markPumpStopped(): Unit = pumpCoordinator.markStopped() -internal fun SabrSessionHolder.wakePump(): Unit = pumpCoordinator.wake() +fun SabrSessionHolder.wakePump(): Unit = pumpCoordinator.wake() -internal fun SabrSessionHolder.pumpWakeVersion(): Long = pumpCoordinator.wakeVersion() +fun SabrSessionHolder.pumpWakeVersion(): Long = pumpCoordinator.wakeVersion() internal suspend fun SabrSessionHolder.awaitPumpWake(observedVersion: Long, timeoutMs: Long): Unit = pumpCoordinator.awaitWake(observedVersion, timeoutMs) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt index 645968ef..351b17fe 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLauncher.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.job import kotlinx.coroutines.launch -internal fun CoroutineScope.launchSabrPump( +fun CoroutineScope.launchSabrPump( pump: SabrSessionPump, registry: SabrSessionRegistry, holder: SabrSessionHolder, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index 849a5068..16531eba 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrPumpLogger { +object SabrPumpLogger { private val logger = LoggerFactory.getLogger(SabrSessionPumpLoop::class.java) fun start(holder: SabrSessionHolder, event: String, request: SabrSegmentRequest?): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt index 15002930..8076b077 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLoopPolicy.kt @@ -1,13 +1,13 @@ package dev.typetype.server.services -internal fun preparePumpEviction(holder: SabrSessionHolder) { +fun preparePumpEviction(holder: SabrSessionHolder) { val retainedPlayHeadMs = (holder.readerTailMs() - SabrPumpPolicy.backBufferMs(holder)).coerceAtLeast(0L) holder.session.setPlayHeadMs(retainedPlayHeadMs) holder.session.evictPlayed() } -internal fun pumpDemandDelayMs(holder: SabrSessionHolder, intervalMs: Long): Long { +fun pumpDemandDelayMs(holder: SabrSessionHolder, intervalMs: Long): Long { val demand = holder.nextSegmentDemand() return SabrPumpPolicy.demandDelayMs( intervalMs, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt index 8a5c6eae..3b4f090d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object SabrPumpPolicy { +object SabrPumpPolicy { const val IDLE_POLL_MS = 100L const val ERROR_RETRY_MS = 1_000L const val MAX_CONSECUTIVE_IO_ERRORS = 5 diff --git a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt index fefd9b29..fae2144b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import kotlin.math.roundToLong -internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) { +class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) { private val startedAtMs = clock() private val protectedResponseGuard = SabrProtectedResponseGuard() private var lastRequestMs = 0L @@ -61,7 +61,7 @@ internal class SabrPumpRuntime(private val clock: () -> Long = System::currentTi holder.session.cachedBytes > SabrPumpPolicy.MAX_AHEAD_BYTES } - internal fun targetReadaheadCushionMs(holder: SabrSessionHolder): Long { + fun targetReadaheadCushionMs(holder: SabrSessionHolder): Long { val baseCushionMs = when { isSeekMode() -> SabrPumpPolicy.SEEK_READAHEAD_CUSHION_MS isStartupBurst() -> SabrPumpPolicy.STARTUP_BURST_READAHEAD_CUSHION_MS diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt index 3e262d83..1dda4386 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentCache.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal class SabrSegmentCache { +class SabrSegmentCache { fun get(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? = holder.cachedSegment(key(holder, request)) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt index 8d8ee50b..dabd7641 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandResolution.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal fun SabrSessionHolder.resolveSegmentDemand( +fun SabrSessionHolder.resolveSegmentDemand( request: SabrSegmentRequest, identity: String, onResolved: (SabrMediaSegment) -> Unit = {}, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt similarity index 84% rename from src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt index f68e0adc..2a8c9f76 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong -internal object SabrSegmentDemandTracker { +object SabrSegmentDemandTracker { private val demands = ConcurrentHashMap() private val order = AtomicLong() @@ -100,7 +100,7 @@ internal object SabrSegmentDemandTracker { ) } -internal fun SabrSessionHolder.requestSegmentDemand( +fun SabrSessionHolder.requestSegmentDemand( request: SabrSegmentRequest, generation: Long = activeGeneration(), registeredAtMs: Long = System.currentTimeMillis(), @@ -111,23 +111,23 @@ internal fun SabrSessionHolder.requestSegmentDemand( } } -internal fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest): Unit = +fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest): Unit = SabrSegmentDemandTracker.clear(this, request) -internal fun SabrSessionHolder.clearSegmentDemands(): Unit = SabrSegmentDemandTracker.clear(this) +fun SabrSessionHolder.clearSegmentDemands(): Unit = SabrSegmentDemandTracker.clear(this) -internal fun SabrSessionHolder.nextSegmentDemand(): SabrSegmentRequest? = SabrSegmentDemandTracker.next(this) +fun SabrSessionHolder.nextSegmentDemand(): SabrSegmentRequest? = SabrSegmentDemandTracker.next(this) -internal fun SabrSessionHolder.segmentDemandIdentity(request: SabrSegmentRequest): String? = +fun SabrSessionHolder.segmentDemandIdentity(request: SabrSegmentRequest): String? = SabrSegmentDemandTracker.identity(this, request) -internal fun SabrSessionHolder.isSegmentDemandActive(request: SabrSegmentRequest, identity: String): Boolean = +fun SabrSessionHolder.isSegmentDemandActive(request: SabrSegmentRequest, identity: String): Boolean = SabrSegmentDemandTracker.isActive(this, request, identity) -internal fun SabrSessionHolder.segmentDemandRegisteredAtMs(request: SabrSegmentRequest, identity: String): Long? = +fun SabrSessionHolder.segmentDemandRegisteredAtMs(request: SabrSegmentRequest, identity: String): Long? = SabrSegmentDemandTracker.registeredAtMs(this, request, identity) -internal fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean = +fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean = SabrSegmentDemandTracker.clear(this, request, identity) -internal fun SabrSessionHolder.pendingSegmentDemandSummary(): String? = SabrSegmentDemandTracker.pendingSummary(this) +fun SabrSessionHolder.pendingSegmentDemandSummary(): String? = SabrSegmentDemandTracker.pendingSummary(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt index 19dbecf3..7da9bde4 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionFactory.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession import java.time.Instant -internal class SabrSessionFactory( +class SabrSessionFactory( private val tokenClient: TypetypeTokenSabrTokenClient, ) { fun create( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt index fb41f20c..dbd9c57b 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionHolder.kt @@ -12,7 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference -internal class SabrSessionHolder( +class SabrSessionHolder( val session: YoutubeSabrSession, val info: YoutubeSabrInfo, val audioFormat: YoutubeSabrFormat, @@ -33,7 +33,7 @@ internal class SabrSessionHolder( private val liveInitializationData = ConcurrentHashMap() private val pendingRefetch = AtomicReference() private val pendingForwardSeek = AtomicReference() - internal val pumpCoordinator = SabrPumpCoordinator() + val pumpCoordinator = SabrPumpCoordinator() private val unauthorizedRefreshAttempted = AtomicBoolean(false) private val expectedLive = AtomicBoolean(false) private val activeGeneration = AtomicLong(initialGeneration.coerceAtLeast(0L)) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt index a11a99fd..6851d04f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionIdentity.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import dev.typetype.server.sabr.SabrAdapter import dev.typetype.server.sabr.YoutubeSabrInfo -internal object SabrSessionIdentity { +object SabrSessionIdentity { fun fresh(info: YoutubeSabrInfo): YoutubeSabrInfo { val cpn = YoutubeParsingHelper.generateContentPlaybackNonce() val url = requireNotNull(info.serverAbrStreamingUrl) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt index aaf6b375..bdf62111 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionKey.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal data class SabrSessionKey( +data class SabrSessionKey( val videoId: String, val userId: String, val audioItag: Int, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt index f0af4faf..54743604 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionMediaFetcher.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory import java.time.Instant -internal object SabrSessionMediaFetcher { +object SabrSessionMediaFetcher { private val logger = LoggerFactory.getLogger(SabrSessionMediaFetcher::class.java) suspend fun fetch( diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt similarity index 68% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt index db7e1bf3..75377809 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPlayerContext.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.YoutubeSabrSession -internal inline fun SabrSessionHolder.withPlayerContext(crossinline block: YoutubeSabrSession.() -> T): T { +inline fun SabrSessionHolder.withPlayerContext(crossinline block: YoutubeSabrSession.() -> T): T { val token = playerContextToken ?: return session.block() return TypetypeYoutubeSessionPoTokenProvider.withToken(token) { session.block() } } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt index f68876c8..7d84aa85 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionProgress.kt @@ -3,11 +3,11 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment): Unit { +fun SabrSessionHolder.markServed(segment: SabrMediaSegment): Unit { markServed(segment, activeGeneration()) } -internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: Long): Unit { +fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: Long): Unit { if (!segment.header.isInitSegment) { observeMediaSegment(segment) val format = if (audioFormat.itag == segment.header.itag) audioFormat else videoFormat @@ -17,7 +17,7 @@ internal fun SabrSessionHolder.markServed(segment: SabrMediaSegment, generation: } } -internal fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { +fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { if (!segment.header.isInitSegment) { observeMediaSegment(segment) val format = if (audioFormat.itag == segment.header.itag) audioFormat else videoFormat @@ -25,11 +25,11 @@ internal fun SabrSessionHolder.markPrepared(segment: SabrMediaSegment): Unit { } } -internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment): Unit { +fun SabrSessionHolder.markServed(segment: CachedSabrSegment): Unit { markServed(segment, activeGeneration()) } -internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation: Long): Unit { +fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation: Long): Unit { if (!segment.init) { val format = if (audioFormat.itag == segment.itag) audioFormat else videoFormat setReaderPosition(format, segment.startMs + segment.durationMs, generation) @@ -38,13 +38,13 @@ internal fun SabrSessionHolder.markServed(segment: CachedSabrSegment, generation } } -internal fun SabrSessionHolder.shouldSend(segment: SabrMediaSegment): Boolean { +fun SabrSessionHolder.shouldSend(segment: SabrMediaSegment): Boolean { if (segment.header.isInitSegment) return true val lastSequence = lastServedSequence(formatForItag(segment.header.itag)) ?: return true return segment.header.sequenceNumber > lastSequence } -internal fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { +fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { if (segment.init) return true val lastSequence = lastServedSequence(formatForItag(segment.itag)) ?: return true return segment.sequence > lastSequence @@ -53,7 +53,7 @@ internal fun SabrSessionHolder.shouldSend(segment: CachedSabrSegment): Boolean { private fun SabrSessionHolder.formatForItag(itag: Int): YoutubeSabrFormat = if (audioFormat.itag == itag) audioFormat else videoFormat -internal fun bothFormatsKnown(holder: SabrSessionHolder): Boolean { +fun bothFormatsKnown(holder: SabrSessionHolder): Boolean { val state = holder.session.streamState return state.getEndSegment(holder.audioFormat) > 0L && state.getEndSegment(holder.videoFormat) > 0L && diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt index 052e08d3..7f3696a9 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPump.kt @@ -7,7 +7,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import java.time.Instant -internal class SabrSessionPump( +class SabrSessionPump( private val segmentCache: SabrSegmentCache? = null, refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle? = { null }, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index 8f541214..e538b266 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -13,7 +13,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession import java.io.IOException -internal class SabrSessionPumpLoop( +class SabrSessionPumpLoop( private val unauthorizedRecovery: SabrUnauthorizedResponseRecovery = SabrUnauthorizedResponseRecovery { null }, private val runtimeFactory: () -> SabrPumpRuntime = { SabrPumpRuntime() }, private val onResolved: (SabrSessionHolder, SabrMediaSegment) -> Unit = { _, _ -> }, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt index 83f64b5e..95ff491c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPurpose.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class SabrSessionPurpose { +enum class SabrSessionPurpose { MANIFEST, PLAYBACK, DOWNLOAD, diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt index 34ef8db2..263bb960 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.time.Instant import java.util.concurrent.ConcurrentHashMap -internal class SabrSessionRegistry { +class SabrSessionRegistry { private val sessions = ConcurrentHashMap() private val sessionsByToken = ConcurrentHashMap() private val mutationLock = Any() diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt similarity index 75% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt index 30cf98c8..b8513a1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionResources.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.releaseResources() { +fun SabrSessionHolder.releaseResources() { clearSegmentDemands() clearInFlightSegmentDemand() SabrPlaybackDiagnostics.clear(this) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt similarity index 59% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt index 77277499..6fddb14f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStorageTypes.kt @@ -2,4 +2,4 @@ package dev.typetype.server.services internal const val SABR_MAX_SEGMENT_MEMORY_BYTES = 64L * 1024L * 1024L -internal data class SabrReaderTrackKey(val generation: Long, val itag: Int) +data class SabrReaderTrackKey(val generation: Long, val itag: Int) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 56ae348a..e631d171 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -16,7 +16,7 @@ import dev.typetype.server.sabr.YoutubeSabrInfo import java.time.Duration import java.time.Instant -internal class SabrSessionStore( +class SabrSessionStore( tokenServiceUrl: String, private val maxSessions: Int = SabrSessionStoreDefaults.maxSessions(), private val idleEviction: Duration = SabrSessionStoreDefaults.idleEviction(), @@ -41,7 +41,7 @@ internal class SabrSessionStore( init { runCatching { NewPipeInitializer.init() } } - internal fun getOrCreate( + fun getOrCreate( videoId: String, userId: String, info: YoutubeSabrInfo, @@ -88,43 +88,43 @@ internal class SabrSessionStore( if (startPump) startPump(holder) return holder } - internal fun startPump(holder: SabrSessionHolder) { + fun startPump(holder: SabrSessionHolder) { scope.launchSabrPump(pump, registry, holder, pumpLoopIntervalMs) } - internal fun warmPlaybackAsync(holder: SabrSessionHolder) { + fun warmPlaybackAsync(holder: SabrSessionHolder) { if (holder.playerTimeMs() > 0L) return if (holder.playbackState() == SabrPlaybackState.REQUESTING || holder.playbackState() == SabrPlaybackState.REPOSITIONING) return scope.launch { fetchMediaAt(holder, holder.playerTimeMs()) } } - internal fun warmInitializationAsync(holder: SabrSessionHolder): Unit = + fun warmInitializationAsync(holder: SabrSessionHolder): Unit = SabrInitializationPolicy.warmFormats(holder.key.audioOnly, holder.audioFormat, holder.videoFormat).forEach { format -> scope.launch { fetchDirectInitialization(holder, format) } } - internal fun lookup(videoId: String, userId: String, audioItag: Int, videoItag: Int): SabrSessionHolder? = + fun lookup(videoId: String, userId: String, audioItag: Int, videoItag: Int): SabrSessionHolder? = registry.get(SabrSessionKey(videoId, userId, audioItag, null, videoItag, 0L)) - internal fun lookupByItag(videoId: String, userId: String, itag: Int): SabrSessionHolder? = + fun lookupByItag(videoId: String, userId: String, itag: Int): SabrSessionHolder? = registry.lookupByItag(videoId, userId, itag) - internal fun lookupByToken(videoId: String, token: String, itag: Int): SabrSessionHolder? = registry.lookupByToken(videoId, token, itag) + fun lookupByToken(videoId: String, token: String, itag: Int): SabrSessionHolder? = registry.lookupByToken(videoId, token, itag) - internal fun lookupByToken(videoId: String, token: String): SabrSessionHolder? = registry.lookupByToken(videoId, token) + fun lookupByToken(videoId: String, token: String): SabrSessionHolder? = registry.lookupByToken(videoId, token) - internal fun lookupByToken(token: String): SabrSessionHolder? = registry.lookupByToken(token) + fun lookupByToken(token: String): SabrSessionHolder? = registry.lookupByToken(token) - internal suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int = 8): Unit = pump.ensureWarmed(holder, maxPumps) + suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int = 8): Unit = pump.ensureWarmed(holder, maxPumps) - internal suspend fun preflightPlayback(holder: SabrSessionHolder, playerTimeMs: Long): Boolean = + suspend fun preflightPlayback(holder: SabrSessionHolder, playerTimeMs: Long): Boolean = warmer.preflight(this, holder, playerTimeMs) - internal suspend fun cachedMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = + suspend fun cachedMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = holder.mediaRequestsAt(playerTimeMs).map { segmentCache.get(holder, it) } .takeIf { cached -> cached.all { it != null } }?.filterNotNull() - internal suspend fun cachedSegment(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? { + suspend fun cachedSegment(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? { holder.session.getCachedSegment(request)?.let { segmentCache.put(holder, it) holder.clearSegmentDemand(request) @@ -133,7 +133,7 @@ internal class SabrSessionStore( return segmentCache.get(holder, request)?.also { holder.clearSegmentDemand(request) } } - internal fun requestSegmentDemand( + fun requestSegmentDemand( holder: SabrSessionHolder, request: SabrSegmentRequest, generation: Long, @@ -142,36 +142,36 @@ internal class SabrSessionStore( startPump(holder) } - internal suspend fun fetchInfo( + suspend fun fetchInfo( videoId: String, startTimeMs: Long = 0L, cachedFirst: Boolean = false, isolatedPlayback: Boolean = false, ): SabrPreparedInfo? = infoFetcher.fetchInfo(videoId, startTimeMs, cachedFirst, isolatedPlayback) - internal suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = + suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) - internal suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) - internal suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) + suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) + suspend fun invalidatePlaybackInfo(videoId: String): Unit = infoFetcher.invalidatePlayback(videoId) - internal suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = + suspend fun recoverProtectedPlaybackInfo(holder: SabrSessionHolder): Unit = infoFetcher.recoverProtectedPlayback( holder.key.videoId, holder.playerContextToken?.visitorData ?: holder.info.visitorData, ) - internal fun refreshVideoPoToken(videoId: String): SabrTokenBundle? = + fun refreshVideoPoToken(videoId: String): SabrTokenBundle? = tokenClient.fetch(videoId, refreshVideo = true) - internal suspend fun fetchSegment( + suspend fun fetchSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, ): SabrMediaSegment? = pump.fetchSegment(holder, request) - internal suspend fun fetchMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = + suspend fun fetchMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = pump.fetchMediaAt(holder, playerTimeMs) - internal suspend fun fetchInitializationData( + suspend fun fetchInitializationData( holder: SabrSessionHolder, format: YoutubeSabrFormat, ): ByteArray? { @@ -207,7 +207,7 @@ internal class SabrSessionStore( registry.clear() } - internal fun release(holder: SabrSessionHolder): Unit = registry.remove(holder) + fun release(holder: SabrSessionHolder): Unit = registry.remove(holder) private suspend fun idleEvictionLoop() { while (true) { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt index 553437b9..ed6291a2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStoreDefaults.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.time.Duration -internal object SabrSessionStoreDefaults { +object SabrSessionStoreDefaults { const val INFO_TIMEOUT_MS = 8_000L const val INFO_ATTEMPTS = 1 const val INFO_RETRY_DELAY_MS = 250L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt index 3d4c0b1a..0b256238 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTimeRequests.kt @@ -3,10 +3,10 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long): List = +fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long): List = mediaRequestsAt(playerTimeMs, activeGeneration()) -internal fun SabrSessionHolder.repositionTargets( +fun SabrSessionHolder.repositionTargets( targets: List, playerTimeMs: Long, generation: Long, @@ -39,7 +39,7 @@ private fun SabrSessionHolder.adjacentWarmedLiveMediaStartMs( } } -internal fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long, generation: Long): List = buildList { +fun SabrSessionHolder.mediaRequestsAt(playerTimeMs: Long, generation: Long): List = buildList { if (isVideoActive() && needsMediaAt(videoFormat, playerTimeMs, generation)) add(mediaRequestAt(videoFormat, playerTimeMs, generation)) if (isAudioActive() && needsMediaAt(audioFormat, playerTimeMs, generation)) add(mediaRequestAt(audioFormat, playerTimeMs, generation)) } diff --git a/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt index 755d4dbd..0a6c52b5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionTokenGenerator.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.security.SecureRandom import java.util.Base64 -internal object SabrSessionTokenGenerator { +object SabrSessionTokenGenerator { private val random = SecureRandom() fun newToken(): String { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt index 47797f98..df118c68 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrStartupBootstrap.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal fun SabrSessionHolder.prepareStartupBootstrapPump(): Boolean { +fun SabrSessionHolder.prepareStartupBootstrapPump(): Boolean { if (expectsLive() || !hasPendingSeek()) return false if (session.requestNumber == 0) { session.streamState.setPlayerTimeMs(0L) diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt index 8465ed22..21d8a181 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequest.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrSession import org.slf4j.LoggerFactory -internal fun YoutubeSabrSession.fetchTargetedSegment( +fun YoutubeSabrSession.fetchTargetedSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, localization: Localization, @@ -24,7 +24,7 @@ internal fun YoutubeSabrSession.fetchTargetedSegment( } result.onFailure { error -> SabrPlaybackDiagnostics.record(holder, request, error.message) - logger.warn( + targetRequestLogger.warn( "sabr_target event=fetch_failed videoId={} itag={} seq={} init={} targetMs={} errorType={} error={}", holder.key.videoId, request.format.itag, @@ -62,6 +62,6 @@ private fun SabrSessionHolder.targetTimeInsideSegment(request: SabrSegmentReques return minOf(startMs + TARGET_SEGMENT_OFFSET_MS, nextStartMs - 1L) } -private val logger = LoggerFactory.getLogger("SabrTargetRequest") +private val targetRequestLogger = LoggerFactory.getLogger("SabrTargetRequest") private const val TARGET_SEGMENT_OFFSET_MS = 1_000L diff --git a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt index 1fef82eb..0aeae523 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrTargetRequestShape.kt @@ -6,7 +6,7 @@ import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrStreamState import org.slf4j.LoggerFactory -internal inline fun withTargetedRequestShape( +inline fun withTargetedRequestShape( holder: SabrSessionHolder, request: SabrSegmentRequest, prepareSession: Boolean = true, @@ -40,19 +40,19 @@ internal inline fun withTargetedRequestShape( } } -private fun SabrSessionHolder.companionFormat(format: YoutubeSabrFormat): YoutubeSabrFormat = +@PublishedApi internal fun SabrSessionHolder.companionFormat(format: YoutubeSabrFormat): YoutubeSabrFormat = if (format.isAudio) videoFormat else audioFormat -private fun SabrSegmentRequest.trackMode(): Int = +@PublishedApi internal fun SabrSegmentRequest.trackMode(): Int = if (format.isAudio) YoutubeSabrStreamState.TRACK_MODE_AUDIO_ONLY else YoutubeSabrStreamState.TRACK_MODE_VIDEO_ONLY -private fun SabrSegmentRequest.targetRange(holder: SabrSessionHolder): SabrBufferedRange = +@PublishedApi internal fun SabrSegmentRequest.targetRange(holder: SabrSessionHolder): SabrBufferedRange = format.bufferedRange(holder, (sequenceNumber - 1).coerceAtLeast(0)) -private fun YoutubeSabrFormat.targetCompanionRange(holder: SabrSessionHolder, targetPlayerTimeMs: Long): SabrBufferedRange = +@PublishedApi internal fun YoutubeSabrFormat.targetCompanionRange(holder: SabrSessionHolder, targetPlayerTimeMs: Long): SabrBufferedRange = bufferedRange(holder, (holder.playbackStartSequence(this, targetPlayerTimeMs) - 1).coerceAtLeast(0)) -private fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, startMs: Long): Long { +@PublishedApi internal fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, startMs: Long): Long { val playerTimeMs = holder.playerTimeMs() val endMs = holder.playbackSegmentEndMs(format, sequenceNumber) if (endMs > startMs && playerTimeMs >= startMs && playerTimeMs < endMs) return playerTimeMs @@ -60,7 +60,7 @@ private fun SabrSegmentRequest.targetPlayerTimeMs(holder: SabrSessionHolder, sta return startMs } -private fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedSequence: Int): SabrBufferedRange { +@PublishedApi internal fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedSequence: Int): SabrBufferedRange { val endMs = holder.playbackSegmentEndMs(this, bufferedSequence) val durationMs = endMs.takeIf { it > 0L } ?: 1L return SabrBufferedRange( @@ -75,8 +75,8 @@ private fun YoutubeSabrFormat.bufferedRange(holder: SabrSessionHolder, bufferedS ) } -private const val SEEK_FORMAT_ORDER_MS = 1_000L +@PublishedApi internal const val SEEK_FORMAT_ORDER_MS = 1_000L private const val TARGET_SEGMENT_OFFSET_MS = 1_000L private const val TIMESCALE = 1_000 -private val logger = LoggerFactory.getLogger("SabrTargetRequestShape") +@PublishedApi internal val logger = LoggerFactory.getLogger("SabrTargetRequestShape") diff --git a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt index db9afa70..d61b5733 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecovery.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrRecoverableException -internal class SabrUnauthorizedResponseRecovery( +class SabrUnauthorizedResponseRecovery( private val refreshPoToken: (SabrSessionHolder) -> SabrTokenBundle?, ) { fun verify(holder: SabrSessionHolder): Unit { diff --git a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt index a6041877..f82ce8bd 100644 --- a/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrWindowSegmentFetcher.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import org.slf4j.LoggerFactory -internal object SabrWindowSegmentFetcher { +object SabrWindowSegmentFetcher { suspend fun fetch( holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt index ec844eb2..b29b7948 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrPoTokenProvider.kt @@ -5,7 +5,7 @@ import dev.typetype.server.sabr.SabrRecoverableException import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrStreamState -internal class TypetypeTokenSabrPoTokenProvider( +class TypetypeTokenSabrPoTokenProvider( private val tokenClient: TypetypeTokenSabrTokenClient, private val initialToken: SabrTokenBundle? = null, ) : SabrPoTokenProvider { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt rename to server-playback/src/main/kotlin/dev/typetype/server/services/YoutubeSessionTokenScope.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrInfoServiceTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt new file mode 100644 index 00000000..8839201d --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/PlaybackTestFixtures.kt @@ -0,0 +1,77 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AudioStreamItem +import dev.typetype.server.models.VideoItem +import dev.typetype.server.models.VideoStreamItem + +fun playbackTestVideoStream( + url: String = "https://example.googlevideo.com/videoplayback?id=1", + codec: String? = "avc1.42c01e", + format: String = "MPEG_4", + resolution: String = "720p", + bitrate: Int = 2000000, + width: Int = 1280, + height: Int = 720, + indexStart: Long = 221, + indexEnd: Long = 893, + initStart: Long = 0, + initEnd: Long = 220, + itag: Int = 137, +): VideoStreamItem = VideoStreamItem( + url = url, + mimeType = "video/mp4", + format = format, + resolution = resolution, + bitrate = bitrate, + codec = codec, + isVideoOnly = true, + itag = itag, + width = width, + height = height, + fps = 30, + contentLength = 10_000_000, + initStart = initStart, + initEnd = initEnd, + indexStart = indexStart, + indexEnd = indexEnd, +) + +fun playbackTestAudioStream( + url: String = "https://example.googlevideo.com/audioplayback?id=1", + codec: String? = "mp4a.40.2", + format: String = "M4A", + bitrate: Int = 128, + indexStart: Long = 100, + indexEnd: Long = 500, + initStart: Long = 0, + initEnd: Long = 99, + audioTrackId: String? = null, + audioTrackName: String? = null, + audioLocale: String? = null, + isOriginal: Boolean = false, + itag: Int = 140, + deliveryMethod: String = "progressive", + manifestUrl: String? = null, + sabrSessionUrl: String? = null, + contentLength: Long = 5_000_000, +): AudioStreamItem = AudioStreamItem( + url = url, + mimeType = "audio/mp4", + format = format, + bitrate = bitrate, + codec = codec, + quality = "medium", + itag = itag, + contentLength = contentLength, + initStart = initStart, + initEnd = initEnd, + indexStart = indexStart, + indexEnd = indexEnd, + audioTrackId = audioTrackId, + audioTrackName = audioTrackName, + audioLocale = audioLocale, + isOriginal = isOriginal, + deliveryMethod = deliveryMethod, + manifestUrl = manifestUrl, + sabrSessionUrl = sabrSessionUrl, +) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt new file mode 100644 index 00000000..a1e9fde4 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.localization.Localization +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState + +class SabrAdaptiveInitializationTest { + @Test + fun `fetches initialization range with the active streaming token`() = runBlocking { + val fixture = fixture(byteArrayOf(1, 2, 3)) + val expected = byteArrayOf(4, 5, 6) + every { + fixture.session.fetchInitializationData( + fixture.format, + any(), + 2_000L, + match { it.contentEquals(fixture.poToken) }, + ) + } returns expected + + val actual = SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L) + + assertArrayEquals(expected, actual) + } + + @Test + fun `returns null when the adaptive range request fails`() = runBlocking { + val fixture = fixture(byteArrayOf(1)) + every { + fixture.session.fetchInitializationData( + fixture.format, + any(), + 2_000L, + any(), + ) + } throws java.io.IOException("range unavailable") + + assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) + } + + @Test + fun `does not request a range without a streaming token`() = runBlocking { + val fixture = fixture(null) + + assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) + verify(exactly = 0) { + fixture.session.fetchInitializationData(any(), any(), any(), any()) + } + } + + private fun fixture(poToken: ByteArray?): Fixture { + val holder = mockk() + val session = mockk() + val state = mockk() + val format = mockk() + every { holder.session } returns session + every { holder.playerContextToken } returns null + every { holder.info } returns mockk() + every { session.streamState } returns state + every { state.poToken } returns poToken + return Fixture(holder, session, format, poToken ?: byteArrayOf()) + } + + private data class Fixture( + val holder: SabrSessionHolder, + val session: YoutubeSabrSession, + val format: YoutubeSabrFormat, + val poToken: ByteArray, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDashManifestBuilderTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandFailurePumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogBackoffTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadRangeTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrDownloadStreamerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt new file mode 100644 index 00000000..2ce89bfa --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt @@ -0,0 +1,27 @@ +package dev.typetype.server.services + +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.YoutubeSabrFormat + +class SabrInitializationPolicyTest { + @Test + fun `audio only warmup excludes video initialization`() { + val audio = mockk() + val video = mockk() + + assertEquals(listOf(audio), SabrInitializationPolicy.warmFormats(true, audio, video)) + assertEquals(listOf(video, audio), SabrInitializationPolicy.warmFormats(false, audio, video)) + } + + @Test + fun `audio only fetch never waits for video initialization`() { + assertFalse(SabrInitializationPolicy.requiresVideoFirst(true, true, 60_000L)) + assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, true, 0L)) + assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, false, 60_000L)) + assertTrue(SabrInitializationPolicy.requiresVideoFirst(false, true, 60_000L)) + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveContinuationRequestTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveFutureRequestTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveMediaNormalizerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackSessionServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt new file mode 100644 index 00000000..70915d31 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -0,0 +1,84 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import java.time.Instant + +class SabrLivePumpStepTest { + @Test + fun `live pump fills below read ahead cushion`() = runTest { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) + var pumps = 0 + + val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + + assertTrue(pumps == 1) + assertFalse(immediate) + } + + @Test + fun `live pump idles above read ahead cushion`() = runTest { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 130_000L) + var pumps = 0 + + val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + + assertTrue(pumps == 0) + assertFalse(immediate) + } + + private fun fixture(playerTimeMs: Long, observedEndMs: Long): Fixture { + val audio = format(140, true) + val video = format(299, false) + val session = mockk(relaxed = true) + every { session.streamState } returns mockk(relaxed = true) + every { session.getCachedSegment(any()) } returns null + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.setPlayerTimeMs(playerTimeMs) + holder.observeMediaSegment(segment(audio.itag, observedEndMs - SEGMENT_DURATION_MS)) + holder.observeMediaSegment(segment(video.itag, observedEndMs - SEGMENT_DURATION_MS)) + return Fixture(holder) + } + + private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { isAudio } returns audio + every { bitrate } returns if (audio) 128_000 else 2_000_000 + every { lastModified } returns 1L + every { xtags } returns null + } + + private fun segment(itag: Int, startMs: Long): SabrMediaSegment { + val header = mockk { + every { this@mockk.itag } returns itag + every { sequenceNumber } returns 100 + every { this@mockk.startMs } returns startMs + every { durationMs } returns SEGMENT_DURATION_MS + every { isInitSegment } returns false + } + return mockk { every { this@mockk.header } returns header } + } + + private data class Fixture(val holder: SabrSessionHolder) + + private companion object { + const val SEGMENT_DURATION_MS = 2_000L + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackCachedSegmentLocatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackDiagnosticsTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackInfoResolverTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt new file mode 100644 index 00000000..1d65b21e --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt @@ -0,0 +1,122 @@ +package dev.typetype.server.services + +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant +import java.util.Base64 + +class SabrPlaybackLiveGapServiceTest { + @Test + fun `serves the next live segment when YouTube skips a sequence`() = runTest { + val video = format(299, isAudio = false) + val audio = format(140, isAudio = true) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { state.isLive } returns true + every { state.getSegmentStartMs(video, 92) } returns 475_000L + val holder = holder(session, audio, video) + val replacement = cachedSegment(299, 93, 480_000L, byteArrayOf(1, 2, 3)) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } answers { + secondArg().takeIf { it.sequenceNumber == 93 }?.let { replacement } + } + every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit + + val result = SabrPlaybackSessionService(store).fetchMedia( + holder = holder, + format = video, + sequence = 92, + timeoutMs = 1_000L, + generation = 0L, + ) as SabrPlaybackSegmentResult.Ready + + assertArrayEquals(byteArrayOf(1, 2, 3), result.bytes) + assertEquals(93, holder.lastServedSequence(video)) + } + + @Test + fun `rejects a following live segment outside the recoverable window`() = runTest { + val video = format(299, isAudio = false) + val audio = format(140, isAudio = true) + val session = mockk(relaxed = true) + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { state.isLive } returns true + every { state.liveHeadTimeMs } returns 31_680_059L + every { state.getSegmentStartMs(video, 6_320) } returns 31_600_059L + every { state.getSegmentEndMs(video, 6_320) } returns 31_605_059L + val holder = holder(session, audio, video) + holder.markExpectedLive() + holder.setLastServedSequence(video.itag, 6_319) + val replacement = cachedSegment(299, 6_335, 31_675_059L, byteArrayOf(1, 2, 3)) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } answers { + secondArg().takeIf { it.sequenceNumber == 6_335 }?.let { replacement } + } + every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit + + val result = SabrPlaybackSessionService(store).fetchMedia( + holder = holder, + format = video, + sequence = 6_320, + timeoutMs = 1_000L, + generation = 0L, + ) + + assertEquals(SabrPlaybackSegmentResult.Retry(holder, "repositioning"), result) + assertEquals(6_319, holder.lastServedSequence(video)) + assertTrue(holder.terminalFailure().orEmpty().contains("live 299 media discontinuity")) + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk(relaxed = true) + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + return format + } + + private fun cachedSegment( + itag: Int, + sequence: Int, + startMs: Long, + bytes: ByteArray, + ): CachedSabrSegment = CachedSabrSegment( + itag = itag, + sequence = sequence, + init = false, + startMs = startMs, + durationMs = 5_000L, + mimeType = "video/mp4", + bytesBase64 = Base64.getEncoder().encodeToString(bytes), + byteLength = bytes.size, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackManifestServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionIsolationTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackSessionServiceTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlayerContextRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt new file mode 100644 index 00000000..0552eddf --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt @@ -0,0 +1,49 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.jupiter.api.Test + +class SabrProtectedContextRecoveryTest { + @Test + fun `refreshes the active rejected context`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns token("rejected") + every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + } + + @Test + fun `keeps a context already refreshed by another recovery`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 0) { tokenClient.fetch("video", forceRefresh = true) } + } + + @Test + fun `refreshes when the current context cannot be read`() { + val tokenClient = mockk() + every { tokenClient.fetch("video") } returns null + every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") + + SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + } + + private fun token(visitorData: String): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "player-$visitorData", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = visitorData, + videoBoundPoToken = "video-$visitorData", + videoBoundPoTokenBytes = byteArrayOf(2), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrProtectedResponseGuardTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpCoordinatorTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpLauncherTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt new file mode 100644 index 00000000..a477f8dc --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrRecoverableException +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +@OptIn(ExperimentalCoroutinesApi::class) +class SabrRecoverablePumpFailureTest { + @Test + fun `recoverable media failure requests a fresh session`() = runTest { + val holder = pumpFailure("Unexpected EOF") + + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertTrue(holder.terminalFailure().orEmpty().startsWith(SABR_RECOVERABLE_FAILURE_PREFIX)) + } + + @Test + fun `local spool failure remains terminal without fresh session recovery`() = runTest { + val holder = pumpFailure("Could not write SABR spool file") + + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertEquals("Could not write SABR spool file", holder.terminalFailure()) + } + + private suspend fun pumpFailure(message: String): SabrSessionHolder { + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + every { session.streamState } returns streamState + every { session.requestNumber } returns 1 + every { session.isComplete } returns false + every { session.cachedBytes } returns 0L + every { streamState.nextRequestPolicy } returns null + every { session.pumpOnceStreaming(any()) } throws SabrRecoverableException(message) + val holder = holder(session) + + SabrSessionPump().pumpLoop({ true }, holder, intervalMs = 0L) + return holder + } + + private fun holder(session: YoutubeSabrSession): SabrSessionHolder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = format(140, true), + videoFormat = format(137, false), + sessionToken = "session-token", + key = SabrSessionKey("video", "user", 140, null, 137, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + return format + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSeekDiagnosticsTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSeekRepositionPumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentCacheTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandTrackerTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionIdentityTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionKeyTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPlayerContextTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPumpLoopTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionPumpTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrSessionRegistryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt new file mode 100644 index 00000000..a8b9a821 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt @@ -0,0 +1,189 @@ +package dev.typetype.server.services + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrSessionTimeRequestsTest { + @Test + fun `playback start sequence is clamped to the known final segment`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 91 + every { state.getEndSegment(audio) } returns 90L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(90, holder.playbackStartSequence(audio, 88_168L)) + } + + @Test + fun `playback start sequence stays valid without an indexed end`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 33 + every { state.getEndSegment(audio) } returns 0L + every { state.getEndSegment(video) } returns 0L + val holder = holder(session, audio, video) + + assertEquals(33, holder.playbackStartSequence(audio, 88_168L)) + } + + @Test + fun `mediaRequestsAt returns active audio and video requests for player time`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 321_601L) } returns 33 + val holder = holder(session, audio, video) + + val requests = holder.mediaRequestsAt(321_601L) + + assertEquals(listOf(137, 140), requests.map { it.format.itag }) + assertEquals(listOf(64, 33), requests.map { it.sequenceNumber }) + } + + @Test + fun `mediaRequestsAt uses mapped video and audio sequences`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 247, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 340_000L) } returns 64 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 340_000L) } returns 35 + val holder = holder(session, audio, video) + + val requests = holder.mediaRequestsAt(340_000L) + + assertEquals(listOf(247, 140), requests.map { it.format.itag }) + assertEquals(listOf(64, 35), requests.map { it.sequenceNumber }) + } + + @Test + fun `mediaRequestsAt excludes inactive tracks`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 137, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 + val holder = holder(session, audio, video) + holder.setActiveTracks(videoActive = true, audioActive = false) + + val requests = holder.mediaRequestsAt(321_601L) + + assertEquals(listOf(137), requests.map { it.format.itag }) + assertEquals(listOf(64), requests.map { it.sequenceNumber }) + } + + @Test + fun `reposition keeps a non adjacent live boundary request pending`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.getSegmentStartMs(audio, 100) } returns 995_000L + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 102, startMs = 995_010L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 102 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 995_000L, generation = 0L) + + assertEquals(listOf(request), missing) + } + + @Test + fun `reposition advances to the next warmed live segment from inside a boundary`() { + val audio = sabrFormat(itag = 140, isAudio = true) + val video = sabrFormat(itag = 248, isAudio = false) + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val observed = mediaSegment(audio.itag, sequence = 101, startMs = 1_000_000L, durationMs = 5_000L) + holder.markExpectedLive() + holder.observeMediaSegment(observed) + every { + session.getCachedSegment(match { + it.format.itag == audio.itag && it.sequenceNumber == 101 + }) + } returns observed + val request = SabrSegmentRequest.media(audio, 100) + + val missing = holder.repositionTargets(listOf(request), playerTimeMs = 996_200L, generation = 0L) + + assertEquals(emptyList(), missing) + assertEquals(1_000_000L, holder.readerPosition(audio)) + } + + private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.isVideo } returns !isAudio + return format + } + + private fun mediaSegment( + itag: Int, + sequence: Int, + startMs: Long, + durationMs: Long = 0L, + ): SabrMediaSegment { + val header = mockk(relaxed = true) + every { header.itag } returns itag + every { header.sequenceNumber } returns sequence + every { header.startMs } returns startMs + every { header.durationMs } returns durationMs + every { header.isInitSegment } returns false + return mockk { every { this@mockk.header } returns header } + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ): SabrSessionHolder = SabrSessionHolder( + session, + mockk(), + audio, + video, + "session", + SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + Instant.now(), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrTransientDemandFailureTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrTransitioningLivePlaybackTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt new file mode 100644 index 00000000..868767af --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AudioStreamItem +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.models.VideoStreamItem + +fun testStreamResponse( + videoOnlyStreams: List = listOf(playbackTestVideoStream()), + audioStreams: List = listOf(playbackTestAudioStream()), + duration: Long = 300, + hlsUrl: String = "", + dashMpdUrl: String = "", +): StreamResponse = StreamResponse( + id = "test-id", + title = "Test Video", + uploaderName = "Test Channel", + uploaderUrl = "https://youtube.com/channel/test", + uploaderAvatarUrl = "", + thumbnailUrl = "", + description = "", + duration = duration, + viewCount = 0, + likeCount = 0, + dislikeCount = 0, + uploadDate = "", + uploaded = 0, + uploaderSubscriberCount = 0, + uploaderVerified = false, + category = "", + license = "", + visibility = "", + tags = emptyList(), + streamType = "video_stream", + isLive = false, + isPostLive = false, + isLiveContent = false, + hasLiveManifest = false, + isShortFormContent = false, + requiresMembership = false, + startPosition = 0L, + streamSegments = emptyList(), + hlsUrl = hlsUrl, + dashMpdUrl = dashMpdUrl, + videoStreams = emptyList(), + audioStreams = audioStreams, + originalAudioTrackId = null, + preferredDefaultAudioTrackId = null, + videoOnlyStreams = videoOnlyStreams, + subtitles = emptyList(), + previewFrames = emptyList(), + sponsorBlockSegments = emptyList(), + relatedStreams = emptyList(), +) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts index 57e97cd8..e1e63a83 100644 --- a/server-token-gateway/build.gradle.kts +++ b/server-token-gateway/build.gradle.kts @@ -6,6 +6,7 @@ dependencies { api(project(":server-sabr")) api(project(":server-core")) implementation(project(":server-db")) + implementation(project(":server-downloader")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("org.slf4j:slf4j-api:2.0.16") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt rename to server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt index 74351ac3..019e2b94 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProvider.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider -internal object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { +object TypetypeYoutubeSessionPoTokenProvider : YoutubeSessionPoTokenProvider { private val scopedToken = ThreadLocal() @Volatile private var authenticatedProvider: YoutubeSessionPoTokenProvider? = null diff --git a/settings.gradle.kts b/settings.gradle.kts index d0468309..96784ac5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,6 +14,7 @@ include(":server-core") include(":server-db") include(":server-downloader") include(":server-domain") +include(":server-playback") include(":server-portability") include(":server-sabr") include(":server-token-gateway") diff --git a/src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt deleted file mode 100644 index e529d61e..00000000 --- a/src/test/kotlin/dev/typetype/server/services/DownloaderGatewayServiceTest.kt +++ /dev/null @@ -1,32 +0,0 @@ -package dev.typetype.server.services - -import com.sun.net.httpserver.HttpServer -import okhttp3.OkHttpClient -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import java.net.InetSocketAddress -import java.util.concurrent.TimeUnit - -class DownloaderGatewayServiceTest { - @Test - fun `stream response ignores regular client read timeout`() { - val upstream = HttpServer.create(InetSocketAddress(0), 0) - upstream.createContext("/events") { exchange -> - val payload = "data: ready\n\n".toByteArray() - exchange.sendResponseHeaders(200, payload.size.toLong()) - Thread.sleep(150) - exchange.responseBody.use { it.write(payload) } - } - upstream.start() - try { - val client = OkHttpClient.Builder().readTimeout(50, TimeUnit.MILLISECONDS).build() - val gateway = DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}", client) - - gateway.openStream("GET", "/events", null, emptyMap(), null).use { response -> - assertEquals("data: ready\n\n", response.body.string()) - } - } finally { - upstream.stop(0) - } - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt index a1e9fde4..e529d61e 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt @@ -1,79 +1,32 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Assertions.assertArrayEquals -import org.junit.jupiter.api.Assertions.assertNull +import com.sun.net.httpserver.HttpServer +import okhttp3.OkHttpClient +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.localization.Localization -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.net.InetSocketAddress +import java.util.concurrent.TimeUnit -class SabrAdaptiveInitializationTest { +class DownloaderGatewayServiceTest { @Test - fun `fetches initialization range with the active streaming token`() = runBlocking { - val fixture = fixture(byteArrayOf(1, 2, 3)) - val expected = byteArrayOf(4, 5, 6) - every { - fixture.session.fetchInitializationData( - fixture.format, - any(), - 2_000L, - match { it.contentEquals(fixture.poToken) }, - ) - } returns expected - - val actual = SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L) - - assertArrayEquals(expected, actual) - } - - @Test - fun `returns null when the adaptive range request fails`() = runBlocking { - val fixture = fixture(byteArrayOf(1)) - every { - fixture.session.fetchInitializationData( - fixture.format, - any(), - 2_000L, - any(), - ) - } throws java.io.IOException("range unavailable") - - assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) - } - - @Test - fun `does not request a range without a streaming token`() = runBlocking { - val fixture = fixture(null) - - assertNull(SabrAdaptiveInitialization.fetchRange(fixture.holder, fixture.format, 2_000L)) - verify(exactly = 0) { - fixture.session.fetchInitializationData(any(), any(), any(), any()) + fun `stream response ignores regular client read timeout`() { + val upstream = HttpServer.create(InetSocketAddress(0), 0) + upstream.createContext("/events") { exchange -> + val payload = "data: ready\n\n".toByteArray() + exchange.sendResponseHeaders(200, payload.size.toLong()) + Thread.sleep(150) + exchange.responseBody.use { it.write(payload) } + } + upstream.start() + try { + val client = OkHttpClient.Builder().readTimeout(50, TimeUnit.MILLISECONDS).build() + val gateway = DownloaderGatewayService("http://127.0.0.1:${upstream.address.port}", client) + + gateway.openStream("GET", "/events", null, emptyMap(), null).use { response -> + assertEquals("data: ready\n\n", response.body.string()) + } + } finally { + upstream.stop(0) } } - - private fun fixture(poToken: ByteArray?): Fixture { - val holder = mockk() - val session = mockk() - val state = mockk() - val format = mockk() - every { holder.session } returns session - every { holder.playerContextToken } returns null - every { holder.info } returns mockk() - every { session.streamState } returns state - every { state.poToken } returns poToken - return Fixture(holder, session, format, poToken ?: byteArrayOf()) - } - - private data class Fixture( - val holder: SabrSessionHolder, - val session: YoutubeSabrSession, - val format: YoutubeSabrFormat, - val poToken: ByteArray, - ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt deleted file mode 100644 index c5c0c8f0..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrFallbackStreamServiceTest.kt +++ /dev/null @@ -1,176 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.testStreamResponse -import dev.typetype.server.models.ExtractionResult -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo - -class SabrFallbackStreamServiceTest { - @Test - fun `enriches empty youtube extraction with token sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) - assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) - assertEquals("fr-FR.4", enriched.originalAudioTrackId) - assertTrue(enriched.audioStreams.single().sabrSessionUrl?.contains("audioTrackId=fr-FR.4") == true) - } - - @Test - fun `replaces classic extraction with prepared sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) - assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } - coVerify(exactly = 0) { tokenSessionClient.fetchPlaybackSession(any()) } - } - - @Test - fun `enriches a live hls extraction with prepared sabr formats`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - val response = testStreamResponse( - videoOnlyStreams = emptyList(), - audioStreams = emptyList(), - hlsUrl = LIVE_HLS_URL, - ).copy(isLive = true, isLiveContent = true, hasLiveManifest = true, streamType = "live_stream") - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val enriched = (result as ExtractionResult.Success).data - assertEquals(LIVE_HLS_URL, enriched.hlsUrl) - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) - } - - @Test - fun `recovers youtube extraction failure with token playback session`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("MWEB player response is not valid") - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession() - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val response = (result as ExtractionResult.Success).data - assertEquals(VIDEO_ID, response.id) - assertEquals("Fallback title", response.title) - assertEquals("Fallback channel", response.uploaderName) - assertEquals("https://example.com/avatar.jpg", response.uploaderAvatarUrl) - assertEquals(3554L, response.duration) - assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), response.audioStreams.map { it.itag }) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } - } - - @Test - fun `preserves token hls when recovering a live stream`() = runTest { - val delegate = mockk() - val sessionStore = mockk() - val tokenSessionClient = mockk() - coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("blocked") - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() - coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession( - isLive = true, - hlsUrl = LIVE_HLS_URL, - ) - val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - - val result = service.getStreamInfo(YOUTUBE_URL) - - val response = (result as ExtractionResult.Success).data - assertEquals(LIVE_HLS_URL, response.hlsUrl) - assertTrue(response.isLive) - assertTrue(response.hasLiveManifest) - } - - private fun preparedInfo(): SabrPreparedInfo { - val video = mockk(relaxed = true) - every { video.isVideo } returns true - every { video.isAudio } returns false - every { video.itag } returns 137 - every { video.mimeType } returns "video/mp4; codecs=\"avc1.640028\"" - every { video.qualityLabel } returns "1080p" - every { video.width } returns 1920 - every { video.height } returns 1080 - every { video.bitrate } returns 4_000_000 - val audio = mockk(relaxed = true) - every { audio.isVideo } returns false - every { audio.isAudio } returns true - every { audio.itag } returns 140 - every { audio.mimeType } returns "audio/mp4; codecs=\"mp4a.40.2\"" - every { audio.audioQuality } returns "AUDIO_QUALITY_MEDIUM" - every { audio.audioTrackId } returns "fr-FR.4" - every { audio.audioTrackDisplayName } returns "French (original)" - every { audio.isOriginalAudio } returns true - every { audio.isAudioDefault } returns true - every { audio.bitrate } returns 129_000 - val info = mockk() - every { info.formats } returns listOf(video, audio) - return SabrPreparedInfo(info, null) - } - - private fun tokenSession(isLive: Boolean = false, hlsUrl: String = ""): TokenYoutubeSession { - val prepared = preparedInfo() - return TokenYoutubeSession( - info = prepared.info, - token = null, - title = "Fallback title", - author = "Fallback channel", - channelId = "channel-id", - channelAvatarUrl = "https://example.com/avatar.jpg", - description = "Fallback description", - durationMs = 3_554_000L, - viewCount = 42L, - thumbnailUrl = "https://example.com/thumb.jpg", - tags = listOf("tag"), - isLive = isLive, - isLiveContent = isLive, - hlsUrl = hlsUrl, - ) - } - - private companion object { - const val VIDEO_ID = "Vj6ReOur1Kk" - const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" - const val LIVE_HLS_URL = "https://example.com/live.m3u8" - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt index 2ce89bfa..c5c0c8f0 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt @@ -1,27 +1,176 @@ package dev.typetype.server.services +import dev.typetype.server.testStreamResponse +import dev.typetype.server.models.ExtractionResult +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every import io.mockk.mockk +import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo -class SabrInitializationPolicyTest { +class SabrFallbackStreamServiceTest { @Test - fun `audio only warmup excludes video initialization`() { - val audio = mockk() - val video = mockk() + fun `enriches empty youtube extraction with token sabr formats`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse(videoOnlyStreams = emptyList(), audioStreams = emptyList()) + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) - assertEquals(listOf(audio), SabrInitializationPolicy.warmFormats(true, audio, video)) - assertEquals(listOf(video, audio), SabrInitializationPolicy.warmFormats(false, audio, video)) + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) + assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) + assertEquals("fr-FR.4", enriched.originalAudioTrackId) + assertTrue(enriched.audioStreams.single().sabrSessionUrl?.contains("audioTrackId=fr-FR.4") == true) + } + + @Test + fun `replaces classic extraction with prepared sabr formats`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + assertEquals("sabr", enriched.videoOnlyStreams.single().deliveryMethod) + assertEquals("sabr", enriched.audioStreams.single().deliveryMethod) + coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { tokenSessionClient.fetchPlaybackSession(any()) } + } + + @Test + fun `enriches a live hls extraction with prepared sabr formats`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + val response = testStreamResponse( + videoOnlyStreams = emptyList(), + audioStreams = emptyList(), + hlsUrl = LIVE_HLS_URL, + ).copy(isLive = true, isLiveContent = true, hasLiveManifest = true, streamType = "live_stream") + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val enriched = (result as ExtractionResult.Success).data + assertEquals(LIVE_HLS_URL, enriched.hlsUrl) + assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + } + + @Test + fun `recovers youtube extraction failure with token playback session`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("MWEB player response is not valid") + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession() + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val response = (result as ExtractionResult.Success).data + assertEquals(VIDEO_ID, response.id) + assertEquals("Fallback title", response.title) + assertEquals("Fallback channel", response.uploaderName) + assertEquals("https://example.com/avatar.jpg", response.uploaderAvatarUrl) + assertEquals(3554L, response.duration) + assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) + assertEquals(listOf(140), response.audioStreams.map { it.itag }) + coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } } @Test - fun `audio only fetch never waits for video initialization`() { - assertFalse(SabrInitializationPolicy.requiresVideoFirst(true, true, 60_000L)) - assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, true, 0L)) - assertFalse(SabrInitializationPolicy.requiresVideoFirst(false, false, 60_000L)) - assertTrue(SabrInitializationPolicy.requiresVideoFirst(false, true, 60_000L)) + fun `preserves token hls when recovering a live stream`() = runTest { + val delegate = mockk() + val sessionStore = mockk() + val tokenSessionClient = mockk() + coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("blocked") + coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() + coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession( + isLive = true, + hlsUrl = LIVE_HLS_URL, + ) + val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) + + val result = service.getStreamInfo(YOUTUBE_URL) + + val response = (result as ExtractionResult.Success).data + assertEquals(LIVE_HLS_URL, response.hlsUrl) + assertTrue(response.isLive) + assertTrue(response.hasLiveManifest) + } + + private fun preparedInfo(): SabrPreparedInfo { + val video = mockk(relaxed = true) + every { video.isVideo } returns true + every { video.isAudio } returns false + every { video.itag } returns 137 + every { video.mimeType } returns "video/mp4; codecs=\"avc1.640028\"" + every { video.qualityLabel } returns "1080p" + every { video.width } returns 1920 + every { video.height } returns 1080 + every { video.bitrate } returns 4_000_000 + val audio = mockk(relaxed = true) + every { audio.isVideo } returns false + every { audio.isAudio } returns true + every { audio.itag } returns 140 + every { audio.mimeType } returns "audio/mp4; codecs=\"mp4a.40.2\"" + every { audio.audioQuality } returns "AUDIO_QUALITY_MEDIUM" + every { audio.audioTrackId } returns "fr-FR.4" + every { audio.audioTrackDisplayName } returns "French (original)" + every { audio.isOriginalAudio } returns true + every { audio.isAudioDefault } returns true + every { audio.bitrate } returns 129_000 + val info = mockk() + every { info.formats } returns listOf(video, audio) + return SabrPreparedInfo(info, null) + } + + private fun tokenSession(isLive: Boolean = false, hlsUrl: String = ""): TokenYoutubeSession { + val prepared = preparedInfo() + return TokenYoutubeSession( + info = prepared.info, + token = null, + title = "Fallback title", + author = "Fallback channel", + channelId = "channel-id", + channelAvatarUrl = "https://example.com/avatar.jpg", + description = "Fallback description", + durationMs = 3_554_000L, + viewCount = 42L, + thumbnailUrl = "https://example.com/thumb.jpg", + tags = listOf("tag"), + isLive = isLive, + isLiveContent = isLive, + hlsUrl = hlsUrl, + ) + } + + private companion object { + const val VIDEO_ID = "Vj6ReOur1Kk" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val LIVE_HLS_URL = "https://example.com/live.m3u8" } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt deleted file mode 100644 index 94fe8c6a..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt +++ /dev/null @@ -1,300 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant - -class SabrLivePlaybackTest { - @AfterEach - fun clearDemands(): Unit = SabrSegmentDemandTracker.clearAll() - - @Test - fun `active live starts from media that the session can retrieve`() { - val fixture = fixture() - listOf(fixture.audio, fixture.video).forEach { format -> - fixture.holder.observeMediaSegment(mockk { - every { header } returns mockk { - every { isInitSegment } returns false - every { itag } returns format.itag - every { sequenceNumber } returns 199 - every { startMs } returns 1_003_000L - every { durationMs } returns 2_000L - } - }) - } - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertTrue(live.active) - assertFalse(live.postLiveDvr) - assertEquals(1_005_000L, live.seekableEndMs) - assertEquals(1_003_000L, fixture.holder.resolvePlaybackStartMs(0L)) - assertEquals(1_005_000L, fixture.holder.resolvePlaybackStartMs(1_100_000L)) - } - - @Test - fun `reported live head wins over sequence based duration estimates`() { - val fixture = fixture() - every { fixture.state.getSegmentEndMs(fixture.video, 200) } returns 9_011_868_000L - every { fixture.state.getBufferedEndMs(fixture.video) } returns 9_011_868_000L - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertEquals(1_005_000L, live.headTimeMs) - assertEquals(1_005_000L, live.seekableEndMs) - } - - @Test - fun `active live maps time from an observed sabr segment for every codec`() { - val fixture = fixture() - every { fixture.state.getMaxSegment(fixture.video) } returns 180 - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns 2_000L - } - val segment = mockk { - every { this@mockk.header } returns header - } - fixture.holder.observeMediaSegment(segment) - - assertEquals(200L, requireNotNull(fixture.holder.livePlaybackSnapshot()).headSequence) - assertEquals(segment, fixture.holder.observedMediaSegment(fixture.video)) - assertEquals(965_000L, segment.header.startMs) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_001L)) - assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) - assertEquals(179, fixture.holder.playbackStartSequence(fixture.video, 964_999L)) - assertEquals(200, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) - every { fixture.state.getMaxSegment(fixture.video) } returns 200 - val nearHead = mockk { - every { this@mockk.header } returns mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 199 - every { startMs } returns 1_003_000L - every { durationMs } returns 2_000L - } - } - fixture.holder.observeMediaSegment(nearHead) - assertEquals(199, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) - } - - @Test - fun `active live derives missing segment duration from the sabr head`() { - val fixture = fixture() - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns -1L - } - val segment = mockk { - every { this@mockk.header } returns header - } - fixture.holder.observeMediaSegment(segment) - - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) - } - - @Test - fun `active live maps time before the head sequence is reported`() { - val fixture = fixture() - every { fixture.session.liveHeadSequenceNumber } returns 0L - every { fixture.state.liveHeadSequenceNumber } returns 0L - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns 2_000L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - } - - @Test - fun `live duration tolerates millisecond drift between media and head timestamps`() { - val fixture = fixture() - every { fixture.state.liveHeadTimeMs } returns 1_004_999L - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 180 - every { startMs } returns 965_000L - every { durationMs } returns -1L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - - assertEquals(2_000L, fixture.holder.playbackSegmentDurationMs(fixture.video, 180)) - assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) - } - - @Test - fun `only the next live media segments wait for production`() { - val fixture = fixture() - - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 202))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 101))) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 102))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 203))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 103))) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 100))) - } - - @Test - fun `advertised live head waits until its media is complete`() { - val fixture = fixture() - every { fixture.state.liveHeadTimeMs } returns 1_025_000L - fixture.holder.observeMediaSegment(mockk { - every { header } returns mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 200 - every { startMs } returns 1_002_000L - every { durationMs } returns 2_000L - } - }) - - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) - } - - @Test - fun `in flight live media waits beyond the reported head`() { - val fixture = fixture() - val request = SabrSegmentRequest.media(fixture.audio, 103) - every { fixture.session.getReadableSegment(request) } returns mockk() - - assertTrue(fixture.holder.isFutureLiveRequest(request)) - } - - @Test - fun `live head can advance beyond the last complete media segment`() { - val fixture = fixture() - val header = mockk { - every { isInitSegment } returns false - every { itag } returns fixture.video.itag - every { sequenceNumber } returns 198 - every { startMs } returns 998_000L - every { durationMs } returns 2_000L - } - fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) - every { fixture.state.liveHeadTimeMs } returns 1_025_000L - fixture.holder.setLastServedSequence(fixture.video.itag, 200) - assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - assertFalse(fixture.holder.isHistoricalLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - } - - @Test - fun `future live demand remains retryable after repeated responses`() { - val fixture = fixture() - val request = SabrSegmentRequest.media(fixture.video, 201) - fixture.holder.requestSegmentDemand(request, registeredAtMs = 0L) - val identity = requireNotNull(fixture.holder.segmentDemandIdentity(request)) - val result = mockk { - every { segmentCount } returns 2 - every { targetTrackSegmentCount } returns 1 - } - val runtime = SabrPumpRuntime { 20_000L } - val wasFutureLiveRequest = fixture.holder.isFutureLiveRequest(request) - every { fixture.state.getMaxSegment(fixture.video) } returns 201 - - repeat(4) { - assertFalse( - SabrDemandAttemptFinisher.finish( - fixture.holder, - request, - identity, - result, - runtime, - wasFutureLiveRequest, - ), - ) - } - - assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) - assertNull(fixture.holder.terminalFailure()) - assertEquals("299:201", fixture.holder.pendingSegmentDemandSummary()) - } - - @Test - fun `post live dvr is finite instead of an active live edge`() { - val fixture = fixture(postLiveDvr = true) - - val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) - - assertFalse(live.active) - assertTrue(live.postLiveDvr) - assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) - } - - private fun fixture(postLiveDvr: Boolean = false): Fixture { - val audio = format(140, true) - val video = format(299, false) - val state = mockk(relaxed = true) - val session = mockk(relaxed = true) - every { session.streamState } returns state - every { session.isLive } returns !postLiveDvr - every { session.isAtLiveEdge } returns !postLiveDvr - every { session.liveHeadSequenceNumber } returns 200L - every { session.getCachedSegment(any()) } returns null - every { session.getReadableSegment(any()) } returns null - every { state.isLive } returns !postLiveDvr - every { state.isPostLiveDvr } returns postLiveDvr - every { state.liveHeadSequenceNumber } returns 200L - every { state.liveHeadTimeMs } returns 1_005_000L - every { state.getMaxSegment(audio) } returns 100 - every { state.getMaxSegment(video) } returns 200 - every { state.getSegmentEndMs(audio, 100) } returns 1_000_000L - every { state.getSegmentEndMs(video, 200) } returns 1_002_000L - every { state.getBufferedEndMs(audio) } returns 1_000_000L - every { state.getBufferedEndMs(video) } returns 1_002_000L - every { state.getMinBufferedEndMs() } returns 1_000_000L - val holder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - return Fixture(holder, session, state, audio, video) - } - - private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 - return format - } - - private data class Fixture( - val holder: SabrSessionHolder, - val session: YoutubeSabrSession, - val state: YoutubeSabrStreamState, - val audio: YoutubeSabrFormat, - val video: YoutubeSabrFormat, - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt index c4877083..94fe8c6a 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLiveProtocolProbeTest.kt @@ -1,170 +1,300 @@ package dev.typetype.server.services -import kotlinx.coroutines.runBlocking +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.SabrMediaHeader +import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.YoutubeSabrFormat -import java.security.MessageDigest +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import java.time.Instant + +class SabrLivePlaybackTest { + @AfterEach + fun clearDemands(): Unit = SabrSegmentDemandTracker.clearAll() -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrLiveProtocolProbeTest { @Test - fun `retrieves every requested live sabr codec`(): Unit = runBlocking { - val videoId = sabrProbeVideoId() - val tokenServiceUrl = sabrProbeTokenServiceUrl() - NewPipeInitializer.init(tokenServiceUrl) - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - try { - val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) - val prepared = SabrInfoFetcher( - tokenClient, - TypetypeTokenYoutubeSessionClient(tokenServiceUrl), - ).fetchInfo(videoId) ?: error("Missing SABR player metadata") - println( - "profile=${prepared.info.profile} clientVersion=${prepared.info.clientVersion} " + - "visitorMatches=${prepared.info.visitorData == prepared.initialToken?.visitorData}" - ) - val audio = prepared.info.formats.first { it.itag == sabrProbeAudioItag() && it.isAudio } - val requestedVideoItags = sabrProbeVideoItags() - val videos = requestedVideoItags.map { itag -> - prepared.info.formats.first { it.itag == itag && it.isVideo } - } - assertEquals(requestedVideoItags, videos.map { it.itag }) - videos.forEach { video -> - probe(store, videoId, prepared, audio, video) - } - } finally { - store.release() + fun `active live starts from media that the session can retrieve`() { + val fixture = fixture() + listOf(fixture.audio, fixture.video).forEach { format -> + fixture.holder.observeMediaSegment(mockk { + every { header } returns mockk { + every { isInitSegment } returns false + every { itag } returns format.itag + every { sequenceNumber } returns 199 + every { startMs } returns 1_003_000L + every { durationMs } returns 2_000L + } + }) } + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertTrue(live.active) + assertFalse(live.postLiveDvr) + assertEquals(1_005_000L, live.seekableEndMs) + assertEquals(1_003_000L, fixture.holder.resolvePlaybackStartMs(0L)) + assertEquals(1_005_000L, fixture.holder.resolvePlaybackStartMs(1_100_000L)) } - private suspend fun probe( - store: SabrSessionStore, - videoId: String, - prepared: SabrPreparedInfo, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - ) { - val label = "${video.codecFamily()}-${video.itag}" - val holder = store.getOrCreate( - videoId = videoId, - userId = "live-protocol-probe-$label", - info = prepared.info, - audioFormat = audio, - videoFormat = video, - initialToken = prepared.initialToken, - startPump = false, - ) - holder.markExpectedLive() - store.ensureWarmed(holder, maxPumps = 8) - val live = requireNotNull(holder.livePlaybackSnapshot()) - val segments = listOfNotNull( - holder.observedMediaSegment(audio), - holder.observedMediaSegment(video), - ) - println( - "$label requests=${holder.session.requestNumber} segments=${segments.size} " + - "headSeq=${live.headSequence} headMs=${live.headTimeMs} startMs=${holder.resolvePlaybackStartMs(0L)}" - ) - println("$label trace=${holder.session.diagnosticTrace}") - assertEquals(setOf(audio.itag, video.itag), segments.map { it.header.itag }.toSet()) - segments.forEach { segment -> - val header = segment.header - val format = if (header.itag == audio.itag) audio else video - val parts = requireNotNull(SabrLiveMediaNormalizer.split(format.mimeType.orEmpty(), segment.data)) { - "$label could not split live ${format.mimeType}" + @Test + fun `reported live head wins over sequence based duration estimates`() { + val fixture = fixture() + every { fixture.state.getSegmentEndMs(fixture.video, 200) } returns 9_011_868_000L + every { fixture.state.getBufferedEndMs(fixture.video) } returns 9_011_868_000L + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertEquals(1_005_000L, live.headTimeMs) + assertEquals(1_005_000L, live.seekableEndMs) + } + + @Test + fun `active live maps time from an observed sabr segment for every codec`() { + val fixture = fixture() + every { fixture.state.getMaxSegment(fixture.video) } returns 180 + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns 2_000L + } + val segment = mockk { + every { this@mockk.header } returns header + } + fixture.holder.observeMediaSegment(segment) + + assertEquals(200L, requireNotNull(fixture.holder.livePlaybackSnapshot()).headSequence) + assertEquals(segment, fixture.holder.observedMediaSegment(fixture.video)) + assertEquals(965_000L, segment.header.startMs) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_001L)) + assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) + assertEquals(179, fixture.holder.playbackStartSequence(fixture.video, 964_999L)) + assertEquals(200, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) + every { fixture.state.getMaxSegment(fixture.video) } returns 200 + val nearHead = mockk { + every { this@mockk.header } returns mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 199 + every { startMs } returns 1_003_000L + every { durationMs } returns 2_000L } - assertTrue(header.sequenceNumber > 0, "$label returned bootstrap media as playable media") - assertTrue(parts.initialization.isNotEmpty(), "$label returned an empty initialization") - assertTrue(parts.media.isNotEmpty(), "$label returned empty media") - assertTrue(holder.liveInitialization(format)?.isNotEmpty() == true, "$label did not retain initialization") - println( - " itag=${header.itag} seq=${header.sequenceNumber} startMs=${header.startMs} " + - "durationMs=${header.durationMs} bytes=${segment.length} sha256=${fingerprint(segment.data)} " + - "boxes=${mp4BoxNames(segment.data)} tfdt=${mp4DecodeTimes(segment.data, header.timeRangeTimescale)}" - ) } + fixture.holder.observeMediaSegment(nearHead) + assertEquals(199, fixture.holder.playbackStartSequence(fixture.video, 1_006_000L)) } - private fun YoutubeSabrFormat.codecFamily(): String { - val mime = mimeType.orEmpty().lowercase() - return when { - "avc1" in mime -> "H.264" - "vp09" in mime || "vp9" in mime -> "VP9" - "av01" in mime -> "AV1" - else -> error("Unsupported probe codec for itag $itag: $mime") + @Test + fun `active live derives missing segment duration from the sabr head`() { + val fixture = fixture() + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns -1L } + val segment = mockk { + every { this@mockk.header } returns header + } + fixture.holder.observeMediaSegment(segment) + + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) + assertEquals(180, fixture.holder.playbackStartSequence(fixture.video, 965_000L)) } - private fun mp4BoxNames(data: ByteArray): String { - val names = mutableListOf() - var offset = 0 - while (offset + 8 <= data.size && names.size < 12) { - val size = ((data[offset].toLong() and 0xff) shl 24) or - ((data[offset + 1].toLong() and 0xff) shl 16) or - ((data[offset + 2].toLong() and 0xff) shl 8) or - (data[offset + 3].toLong() and 0xff) - val type = String(data, offset + 4, 4, Charsets.US_ASCII) - if (size < 8L || size > data.size - offset) break - names += "$type:$size" - offset += size.toInt() + @Test + fun `active live maps time before the head sequence is reported`() { + val fixture = fixture() + every { fixture.session.liveHeadSequenceNumber } returns 0L + every { fixture.state.liveHeadSequenceNumber } returns 0L + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns 2_000L } - return names.joinToString(",") + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) } - private fun fingerprint(data: ByteArray): String = MessageDigest.getInstance("SHA-256") - .digest(data) - .take(6) - .joinToString("") { "%02x".format(it) } - - private fun mp4DecodeTimes(data: ByteArray, timescale: Int): String { - if (timescale <= 0) return "unavailable" - val decodeTimes = mutableListOf() - collectDecodeTimes(data, 0, data.size, decodeTimes) - if (decodeTimes.isEmpty()) return "none" - val firstMs = decodeTimes.first() * 1_000L / timescale - val lastMs = decodeTimes.last() * 1_000L / timescale - return "count=${decodeTimes.size},firstMs=$firstMs,lastMs=$lastMs" + @Test + fun `live duration tolerates millisecond drift between media and head timestamps`() { + val fixture = fixture() + every { fixture.state.liveHeadTimeMs } returns 1_004_999L + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 180 + every { startMs } returns 965_000L + every { durationMs } returns -1L + } + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + + assertEquals(2_000L, fixture.holder.playbackSegmentDurationMs(fixture.video, 180)) + assertEquals(195, fixture.holder.playbackStartSequence(fixture.video, 995_000L)) } - private fun collectDecodeTimes(data: ByteArray, start: Int, end: Int, output: MutableList) { - var offset = start - while (offset + 8 <= end) { - val size = data.readUnsignedInt(offset) - if (size < 8L || size > end - offset) return - val type = String(data, offset + 4, 4, Charsets.US_ASCII) - val payloadStart = offset + 8 - val boxEnd = offset + size.toInt() - when (type) { - "moof", "traf" -> collectDecodeTimes(data, payloadStart, boxEnd, output) - "tfdt" -> data.readTfdt(payloadStart, boxEnd)?.let(output::add) + @Test + fun `only the next live media segments wait for production`() { + val fixture = fixture() + + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 202))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 101))) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 102))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 203))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 103))) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.audio, 100))) + } + + @Test + fun `advertised live head waits until its media is complete`() { + val fixture = fixture() + every { fixture.state.liveHeadTimeMs } returns 1_025_000L + fixture.holder.observeMediaSegment(mockk { + every { header } returns mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 200 + every { startMs } returns 1_002_000L + every { durationMs } returns 2_000L } - offset = boxEnd + }) + + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 200))) + } + + @Test + fun `in flight live media waits beyond the reported head`() { + val fixture = fixture() + val request = SabrSegmentRequest.media(fixture.audio, 103) + every { fixture.session.getReadableSegment(request) } returns mockk() + + assertTrue(fixture.holder.isFutureLiveRequest(request)) + } + + @Test + fun `live head can advance beyond the last complete media segment`() { + val fixture = fixture() + val header = mockk { + every { isInitSegment } returns false + every { itag } returns fixture.video.itag + every { sequenceNumber } returns 198 + every { startMs } returns 998_000L + every { durationMs } returns 2_000L } + fixture.holder.observeMediaSegment(mockk { every { this@mockk.header } returns header }) + every { fixture.state.liveHeadTimeMs } returns 1_025_000L + fixture.holder.setLastServedSequence(fixture.video.itag, 200) + assertTrue(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) + assertFalse(fixture.holder.isHistoricalLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) } - private fun ByteArray.readTfdt(offset: Int, end: Int): Long? { - if (offset + 8 > end) return null - return if (this[offset].toInt() == 1) { - if (offset + 12 > end) null else readUnsignedLong(offset + 4) - } else { - readUnsignedInt(offset + 4) + @Test + fun `future live demand remains retryable after repeated responses`() { + val fixture = fixture() + val request = SabrSegmentRequest.media(fixture.video, 201) + fixture.holder.requestSegmentDemand(request, registeredAtMs = 0L) + val identity = requireNotNull(fixture.holder.segmentDemandIdentity(request)) + val result = mockk { + every { segmentCount } returns 2 + every { targetTrackSegmentCount } returns 1 + } + val runtime = SabrPumpRuntime { 20_000L } + val wasFutureLiveRequest = fixture.holder.isFutureLiveRequest(request) + every { fixture.state.getMaxSegment(fixture.video) } returns 201 + + repeat(4) { + assertFalse( + SabrDemandAttemptFinisher.finish( + fixture.holder, + request, + identity, + result, + runtime, + wasFutureLiveRequest, + ), + ) } + + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) + assertNull(fixture.holder.terminalFailure()) + assertEquals("299:201", fixture.holder.pendingSegmentDemandSummary()) + } + + @Test + fun `post live dvr is finite instead of an active live edge`() { + val fixture = fixture(postLiveDvr = true) + + val live = requireNotNull(fixture.holder.livePlaybackSnapshot()) + + assertFalse(live.active) + assertTrue(live.postLiveDvr) + assertFalse(fixture.holder.isFutureLiveRequest(SabrSegmentRequest.media(fixture.video, 201))) } - private fun ByteArray.readUnsignedInt(offset: Int): Long = - ((this[offset].toLong() and 0xff) shl 24) or - ((this[offset + 1].toLong() and 0xff) shl 16) or - ((this[offset + 2].toLong() and 0xff) shl 8) or - (this[offset + 3].toLong() and 0xff) + private fun fixture(postLiveDvr: Boolean = false): Fixture { + val audio = format(140, true) + val video = format(299, false) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns !postLiveDvr + every { session.isAtLiveEdge } returns !postLiveDvr + every { session.liveHeadSequenceNumber } returns 200L + every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } returns null + every { state.isLive } returns !postLiveDvr + every { state.isPostLiveDvr } returns postLiveDvr + every { state.liveHeadSequenceNumber } returns 200L + every { state.liveHeadTimeMs } returns 1_005_000L + every { state.getMaxSegment(audio) } returns 100 + every { state.getMaxSegment(video) } returns 200 + every { state.getSegmentEndMs(audio, 100) } returns 1_000_000L + every { state.getSegmentEndMs(video, 200) } returns 1_002_000L + every { state.getBufferedEndMs(audio) } returns 1_000_000L + every { state.getBufferedEndMs(video) } returns 1_002_000L + every { state.getMinBufferedEndMs() } returns 1_000_000L + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + return Fixture(holder, session, state, audio, video) + } - private fun ByteArray.readUnsignedLong(offset: Int): Long { - var value = 0L - repeat(8) { index -> value = (value shl 8) or (this[offset + index].toLong() and 0xff) } - return value + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.itag } returns itag + every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 + return format } + + private data class Fixture( + val holder: SabrSessionHolder, + val session: YoutubeSabrSession, + val state: YoutubeSabrStreamState, + val audio: YoutubeSabrFormat, + val video: YoutubeSabrFormat, + ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt index 70915d31..c4877083 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrLivePumpStepTest.kt @@ -1,84 +1,170 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertFalse +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment +import org.junit.jupiter.api.condition.EnabledIfSystemProperty import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import java.time.Instant +import java.security.MessageDigest -class SabrLivePumpStepTest { +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrLiveProtocolProbeTest { @Test - fun `live pump fills below read ahead cushion`() = runTest { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) - var pumps = 0 - - val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } - - assertTrue(pumps == 1) - assertFalse(immediate) + fun `retrieves every requested live sabr codec`(): Unit = runBlocking { + val videoId = sabrProbeVideoId() + val tokenServiceUrl = sabrProbeTokenServiceUrl() + NewPipeInitializer.init(tokenServiceUrl) + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + try { + val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) + val prepared = SabrInfoFetcher( + tokenClient, + TypetypeTokenYoutubeSessionClient(tokenServiceUrl), + ).fetchInfo(videoId) ?: error("Missing SABR player metadata") + println( + "profile=${prepared.info.profile} clientVersion=${prepared.info.clientVersion} " + + "visitorMatches=${prepared.info.visitorData == prepared.initialToken?.visitorData}" + ) + val audio = prepared.info.formats.first { it.itag == sabrProbeAudioItag() && it.isAudio } + val requestedVideoItags = sabrProbeVideoItags() + val videos = requestedVideoItags.map { itag -> + prepared.info.formats.first { it.itag == itag && it.isVideo } + } + assertEquals(requestedVideoItags, videos.map { it.itag }) + videos.forEach { video -> + probe(store, videoId, prepared, audio, video) + } + } finally { + store.release() + } } - @Test - fun `live pump idles above read ahead cushion`() = runTest { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 130_000L) - var pumps = 0 + private suspend fun probe( + store: SabrSessionStore, + videoId: String, + prepared: SabrPreparedInfo, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ) { + val label = "${video.codecFamily()}-${video.itag}" + val holder = store.getOrCreate( + videoId = videoId, + userId = "live-protocol-probe-$label", + info = prepared.info, + audioFormat = audio, + videoFormat = video, + initialToken = prepared.initialToken, + startPump = false, + ) + holder.markExpectedLive() + store.ensureWarmed(holder, maxPumps = 8) + val live = requireNotNull(holder.livePlaybackSnapshot()) + val segments = listOfNotNull( + holder.observedMediaSegment(audio), + holder.observedMediaSegment(video), + ) + println( + "$label requests=${holder.session.requestNumber} segments=${segments.size} " + + "headSeq=${live.headSequence} headMs=${live.headTimeMs} startMs=${holder.resolvePlaybackStartMs(0L)}" + ) + println("$label trace=${holder.session.diagnosticTrace}") + assertEquals(setOf(audio.itag, video.itag), segments.map { it.header.itag }.toSet()) + segments.forEach { segment -> + val header = segment.header + val format = if (header.itag == audio.itag) audio else video + val parts = requireNotNull(SabrLiveMediaNormalizer.split(format.mimeType.orEmpty(), segment.data)) { + "$label could not split live ${format.mimeType}" + } + assertTrue(header.sequenceNumber > 0, "$label returned bootstrap media as playable media") + assertTrue(parts.initialization.isNotEmpty(), "$label returned an empty initialization") + assertTrue(parts.media.isNotEmpty(), "$label returned empty media") + assertTrue(holder.liveInitialization(format)?.isNotEmpty() == true, "$label did not retain initialization") + println( + " itag=${header.itag} seq=${header.sequenceNumber} startMs=${header.startMs} " + + "durationMs=${header.durationMs} bytes=${segment.length} sha256=${fingerprint(segment.data)} " + + "boxes=${mp4BoxNames(segment.data)} tfdt=${mp4DecodeTimes(segment.data, header.timeRangeTimescale)}" + ) + } + } - val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } + private fun YoutubeSabrFormat.codecFamily(): String { + val mime = mimeType.orEmpty().lowercase() + return when { + "avc1" in mime -> "H.264" + "vp09" in mime || "vp9" in mime -> "VP9" + "av01" in mime -> "AV1" + else -> error("Unsupported probe codec for itag $itag: $mime") + } + } - assertTrue(pumps == 0) - assertFalse(immediate) + private fun mp4BoxNames(data: ByteArray): String { + val names = mutableListOf() + var offset = 0 + while (offset + 8 <= data.size && names.size < 12) { + val size = ((data[offset].toLong() and 0xff) shl 24) or + ((data[offset + 1].toLong() and 0xff) shl 16) or + ((data[offset + 2].toLong() and 0xff) shl 8) or + (data[offset + 3].toLong() and 0xff) + val type = String(data, offset + 4, 4, Charsets.US_ASCII) + if (size < 8L || size > data.size - offset) break + names += "$type:$size" + offset += size.toInt() + } + return names.joinToString(",") } - private fun fixture(playerTimeMs: Long, observedEndMs: Long): Fixture { - val audio = format(140, true) - val video = format(299, false) - val session = mockk(relaxed = true) - every { session.streamState } returns mockk(relaxed = true) - every { session.getCachedSegment(any()) } returns null - val holder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session-token", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - holder.setPlayerTimeMs(playerTimeMs) - holder.observeMediaSegment(segment(audio.itag, observedEndMs - SEGMENT_DURATION_MS)) - holder.observeMediaSegment(segment(video.itag, observedEndMs - SEGMENT_DURATION_MS)) - return Fixture(holder) + private fun fingerprint(data: ByteArray): String = MessageDigest.getInstance("SHA-256") + .digest(data) + .take(6) + .joinToString("") { "%02x".format(it) } + + private fun mp4DecodeTimes(data: ByteArray, timescale: Int): String { + if (timescale <= 0) return "unavailable" + val decodeTimes = mutableListOf() + collectDecodeTimes(data, 0, data.size, decodeTimes) + if (decodeTimes.isEmpty()) return "none" + val firstMs = decodeTimes.first() * 1_000L / timescale + val lastMs = decodeTimes.last() * 1_000L / timescale + return "count=${decodeTimes.size},firstMs=$firstMs,lastMs=$lastMs" } - private fun format(itag: Int, audio: Boolean): YoutubeSabrFormat = mockk { - every { this@mockk.itag } returns itag - every { isAudio } returns audio - every { bitrate } returns if (audio) 128_000 else 2_000_000 - every { lastModified } returns 1L - every { xtags } returns null + private fun collectDecodeTimes(data: ByteArray, start: Int, end: Int, output: MutableList) { + var offset = start + while (offset + 8 <= end) { + val size = data.readUnsignedInt(offset) + if (size < 8L || size > end - offset) return + val type = String(data, offset + 4, 4, Charsets.US_ASCII) + val payloadStart = offset + 8 + val boxEnd = offset + size.toInt() + when (type) { + "moof", "traf" -> collectDecodeTimes(data, payloadStart, boxEnd, output) + "tfdt" -> data.readTfdt(payloadStart, boxEnd)?.let(output::add) + } + offset = boxEnd + } } - private fun segment(itag: Int, startMs: Long): SabrMediaSegment { - val header = mockk { - every { this@mockk.itag } returns itag - every { sequenceNumber } returns 100 - every { this@mockk.startMs } returns startMs - every { durationMs } returns SEGMENT_DURATION_MS - every { isInitSegment } returns false + private fun ByteArray.readTfdt(offset: Int, end: Int): Long? { + if (offset + 8 > end) return null + return if (this[offset].toInt() == 1) { + if (offset + 12 > end) null else readUnsignedLong(offset + 4) + } else { + readUnsignedInt(offset + 4) } - return mockk { every { this@mockk.header } returns header } } - private data class Fixture(val holder: SabrSessionHolder) + private fun ByteArray.readUnsignedInt(offset: Int): Long = + ((this[offset].toLong() and 0xff) shl 24) or + ((this[offset + 1].toLong() and 0xff) shl 16) or + ((this[offset + 2].toLong() and 0xff) shl 8) or + (this[offset + 3].toLong() and 0xff) - private companion object { - const val SEGMENT_DURATION_MS = 2_000L + private fun ByteArray.readUnsignedLong(offset: Int): Long { + var value = 0L + repeat(8) { index -> value = (value shl 8) or (this[offset + index].toLong() and 0xff) } + return value } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt deleted file mode 100644 index d59079db..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackInitializationFailureTest.kt +++ /dev/null @@ -1,204 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.routes.SabrPlaybackRecovery -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant - -@OptIn(ExperimentalCoroutinesApi::class) -class SabrPlaybackInitializationFailureTest { - @Test - fun `initialization timeout becomes a recoverable terminal failure`() = runTest { - val audio = format(140, isAudio = true) - val video = format(271, isAudio = false) - val info = mockk() - val prepared = SabrPreparedInfo(info, token()) - val holder = holder(audio, video) - val store = mockk() - every { - store.getOrCreate( - "video", - "user", - info, - audio, - video, - prepared.initialToken, - 475_058L, - false, - SabrSessionPurpose.PLAYBACK, - false, - 0L, - ) - } returns holder - coEvery { store.fetchInitializationData(holder, video) } coAnswers { - delay(Long.MAX_VALUE) - null - } - coEvery { store.fetchInitializationData(holder, audio) } coAnswers { - delay(Long.MAX_VALUE) - null - } - coEvery { store.invalidatePlaybackInfo("video") } returns Unit - - val result = SabrPlaybackSessionService(store).prepare( - "video", - "user", - prepared, - audio, - video, - 475_058L, - ) - - assertFalse(result.ready) - assertEquals(6_000L, testScheduler.currentTime) - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) - verify(exactly = 0) { store.startPump(any()) } - coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } - } - - @Test - fun `missing required initialization terminates before starting pump`() = runBlocking { - val audio = format(140, isAudio = true) - val video = format(271, isAudio = false) - val info = mockk() - val prepared = SabrPreparedInfo(info, token()) - val holder = holder(audio, video) - val store = mockk() - every { - store.getOrCreate( - "video", - "user", - info, - audio, - video, - prepared.initialToken, - 475_058L, - false, - SabrSessionPurpose.PLAYBACK, - false, - 0L, - ) - } returns holder - coEvery { store.fetchInitializationData(holder, video) } returns null - coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1) - coEvery { store.invalidatePlaybackInfo("video") } returns Unit - - val result = SabrPlaybackSessionService(store).prepare( - "video", - "user", - prepared, - audio, - video, - 475_058L, - ) - - assertFalse(result.ready) - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertEquals( - "$SABR_RECOVERABLE_FAILURE_PREFIX SABR initialization unavailable for video:271", - holder.terminalFailure(), - ) - assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) - verify(exactly = 0) { store.startPump(any()) } - coVerify(exactly = 1) { store.invalidatePlaybackInfo("video") } - coVerify(exactly = 1) { store.fetchInitializationData(holder, video) } - coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } - } - - @Test - fun `audio only preparation does not require video initialization`() = runBlocking { - val audio = format(140, isAudio = true) - val video = format(271, isAudio = false) - val info = mockk() - val prepared = SabrPreparedInfo(info, token()) - val holder = holder(audio, video) - val store = mockk() - every { - store.getOrCreate( - "video", - "user", - info, - audio, - video, - prepared.initialToken, - 0L, - false, - SabrSessionPurpose.PLAYBACK, - true, - 0L, - ) - } returns holder - coEvery { store.fetchInitializationData(holder, video) } returns null - coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1) - every { store.startPump(holder) } returns Unit - - SabrPlaybackSessionService(store).prepare( - "video", - "user", - prepared, - audio, - video, - 0L, - audioOnly = true, - ) - - assertEquals(null, holder.terminalFailure()) - verify(exactly = 1) { store.startPump(holder) } - coVerify(exactly = 0) { store.fetchInitializationData(holder, video) } - coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } - } - - private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): SabrSessionHolder { - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { session.getCachedSegment(any()) } returns null - every { session.isBeyondEnd(any()) } returns false - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.setSelectVideoFormatBeforeAudio(any()) } returns Unit - every { state.getMinBufferedEndMs() } returns 0L - return SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session-token", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) - } - - private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.audioTrackId } returns null - every { format.mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" - every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 - return format - } - - private fun token(): SabrTokenBundle = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = "visitor-token", - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = "visitor-data", - videoBoundPoToken = "video-token", - videoBoundPoTokenBytes = byteArrayOf(2), - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt index 1d65b21e..d59079db 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt @@ -1,122 +1,204 @@ package dev.typetype.server.services +import dev.typetype.server.routes.SabrPlaybackRecovery import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant -import java.util.Base64 -class SabrPlaybackLiveGapServiceTest { +@OptIn(ExperimentalCoroutinesApi::class) +class SabrPlaybackInitializationFailureTest { @Test - fun `serves the next live segment when YouTube skips a sequence`() = runTest { - val video = format(299, isAudio = false) + fun `initialization timeout becomes a recoverable terminal failure`() = runTest { val audio = format(140, isAudio = true) - val session = mockk(relaxed = true) - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { session.isLive } returns true - every { state.isLive } returns true - every { state.getSegmentStartMs(video, 92) } returns 475_000L - val holder = holder(session, audio, video) - val replacement = cachedSegment(299, 93, 480_000L, byteArrayOf(1, 2, 3)) + val video = format(271, isAudio = false) + val info = mockk() + val prepared = SabrPreparedInfo(info, token()) + val holder = holder(audio, video) val store = mockk() - coEvery { store.cachedSegment(holder, any()) } answers { - secondArg().takeIf { it.sequenceNumber == 93 }?.let { replacement } + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 475_058L, + false, + SabrSessionPurpose.PLAYBACK, + false, + 0L, + ) + } returns holder + coEvery { store.fetchInitializationData(holder, video) } coAnswers { + delay(Long.MAX_VALUE) + null + } + coEvery { store.fetchInitializationData(holder, audio) } coAnswers { + delay(Long.MAX_VALUE) + null } - every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit + coEvery { store.invalidatePlaybackInfo("video") } returns Unit - val result = SabrPlaybackSessionService(store).fetchMedia( - holder = holder, - format = video, - sequence = 92, - timeoutMs = 1_000L, - generation = 0L, - ) as SabrPlaybackSegmentResult.Ready + val result = SabrPlaybackSessionService(store).prepare( + "video", + "user", + prepared, + audio, + video, + 475_058L, + ) - assertArrayEquals(byteArrayOf(1, 2, 3), result.bytes) - assertEquals(93, holder.lastServedSequence(video)) + assertFalse(result.ready) + assertEquals(6_000L, testScheduler.currentTime) + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) + verify(exactly = 0) { store.startPump(any()) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } @Test - fun `rejects a following live segment outside the recoverable window`() = runTest { - val video = format(299, isAudio = false) + fun `missing required initialization terminates before starting pump`() = runBlocking { val audio = format(140, isAudio = true) - val session = mockk(relaxed = true) - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { session.isLive } returns true - every { state.isLive } returns true - every { state.liveHeadTimeMs } returns 31_680_059L - every { state.getSegmentStartMs(video, 6_320) } returns 31_600_059L - every { state.getSegmentEndMs(video, 6_320) } returns 31_605_059L - val holder = holder(session, audio, video) - holder.markExpectedLive() - holder.setLastServedSequence(video.itag, 6_319) - val replacement = cachedSegment(299, 6_335, 31_675_059L, byteArrayOf(1, 2, 3)) + val video = format(271, isAudio = false) + val info = mockk() + val prepared = SabrPreparedInfo(info, token()) + val holder = holder(audio, video) val store = mockk() - coEvery { store.cachedSegment(holder, any()) } answers { - secondArg().takeIf { it.sequenceNumber == 6_335 }?.let { replacement } - } - every { store.requestSegmentDemand(holder, any(), 0L) } returns Unit + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 475_058L, + false, + SabrSessionPurpose.PLAYBACK, + false, + 0L, + ) + } returns holder + coEvery { store.fetchInitializationData(holder, video) } returns null + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1) + coEvery { store.invalidatePlaybackInfo("video") } returns Unit - val result = SabrPlaybackSessionService(store).fetchMedia( - holder = holder, - format = video, - sequence = 6_320, - timeoutMs = 1_000L, - generation = 0L, + val result = SabrPlaybackSessionService(store).prepare( + "video", + "user", + prepared, + audio, + video, + 475_058L, ) - assertEquals(SabrPlaybackSegmentResult.Retry(holder, "repositioning"), result) - assertEquals(6_319, holder.lastServedSequence(video)) - assertTrue(holder.terminalFailure().orEmpty().contains("live 299 media discontinuity")) + assertFalse(result.ready) + assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) + assertEquals( + "$SABR_RECOVERABLE_FAILURE_PREFIX SABR initialization unavailable for video:271", + holder.terminalFailure(), + ) + assertEquals("retry_fresh_session", SabrPlaybackRecovery(store).action(holder)) + verify(exactly = 0) { store.startPump(any()) } + coVerify(exactly = 1) { store.invalidatePlaybackInfo("video") } + coVerify(exactly = 1) { store.fetchInitializationData(holder, video) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } } - private fun holder( - session: YoutubeSabrSession, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - ): SabrSessionHolder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = audio, - videoFormat = video, - sessionToken = "session-token", - key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - lastRequestAt = Instant.EPOCH, - ) + @Test + fun `audio only preparation does not require video initialization`() = runBlocking { + val audio = format(140, isAudio = true) + val video = format(271, isAudio = false) + val info = mockk() + val prepared = SabrPreparedInfo(info, token()) + val holder = holder(audio, video) + val store = mockk() + every { + store.getOrCreate( + "video", + "user", + info, + audio, + video, + prepared.initialToken, + 0L, + false, + SabrSessionPurpose.PLAYBACK, + true, + 0L, + ) + } returns holder + coEvery { store.fetchInitializationData(holder, video) } returns null + coEvery { store.fetchInitializationData(holder, audio) } returns byteArrayOf(1) + every { store.startPump(holder) } returns Unit + + SabrPlaybackSessionService(store).prepare( + "video", + "user", + prepared, + audio, + video, + 0L, + audioOnly = true, + ) + + assertEquals(null, holder.terminalFailure()) + verify(exactly = 1) { store.startPump(holder) } + coVerify(exactly = 0) { store.fetchInitializationData(holder, video) } + coVerify(exactly = 1) { store.fetchInitializationData(holder, audio) } + } + + private fun holder(audio: YoutubeSabrFormat, video: YoutubeSabrFormat): SabrSessionHolder { + val session = mockk() + val state = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.isBeyondEnd(any()) } returns false + every { state.setActiveTrackTypes(any(), any()) } returns Unit + every { state.setSelectVideoFormatBeforeAudio(any()) } returns Unit + every { state.getMinBufferedEndMs() } returns 0L + return SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + } private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk(relaxed = true) + val format = mockk() every { format.itag } returns itag every { format.isAudio } returns isAudio + every { format.audioTrackId } returns null every { format.mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 return format } - private fun cachedSegment( - itag: Int, - sequence: Int, - startMs: Long, - bytes: ByteArray, - ): CachedSabrSegment = CachedSabrSegment( - itag = itag, - sequence = sequence, - init = false, - startMs = startMs, - durationMs = 5_000L, - mimeType = "video/mp4", - bytesBase64 = Base64.getEncoder().encodeToString(bytes), - byteLength = bytes.size, + private fun token(): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "visitor-token", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = "visitor-data", + videoBoundPoToken = "video-token", + videoBoundPoTokenBytes = byteArrayOf(2), ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt deleted file mode 100644 index 2ce9c2a1..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrPreparedInfoCacheTest.kt +++ /dev/null @@ -1,229 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.coEvery -import io.mockk.coVerify -import io.mockk.mockk -import io.mockk.verify -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertSame -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.YoutubeSabrClientProfile -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import java.time.Duration - -class SabrPreparedInfoCacheTest { - @Test - fun `prepared info reports audio and video only when both are present`() { - assertFalse(preparedInfo(listOf(format(isAudio = false))).hasAudioAndVideoFormats()) - assertFalse(preparedInfo(listOf(format(isAudio = true))).hasAudioAndVideoFormats()) - assertTrue(preparedInfo(listOf(format(isAudio = true), format(isAudio = false))).hasAudioAndVideoFormats()) - } - - @Test - fun `cache can remove incomplete prepared info`() { - val cache = SabrPreparedInfoCache() - cache.put("video", 120_000L, preparedInfo(listOf(format(isAudio = false)))) - - cache.remove("video", 120_000L) - - assertFalse(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) - } - - @Test - fun `cache separates prepared info by start window`() { - val cache = SabrPreparedInfoCache() - cache.put("video", 120_000L, preparedInfo(listOf(format(isAudio = true), format(isAudio = false)))) - - assertTrue(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) - assertFalse(cache.get("video", 340_000L)?.hasAudioAndVideoFormats() == true) - } - - @Test - fun `cache removes every start window for a video`() { - val cache = SabrPreparedInfoCache() - val prepared = preparedInfo(listOf(format(isAudio = true), format(isAudio = false))) - cache.put("video", 120_000L, prepared) - cache.put("video", 340_000L, prepared) - - cache.remove("video") - - assertFalse(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) - assertFalse(cache.get("video", 340_000L)?.hasAudioAndVideoFormats() == true) - } - - @Test - fun `cache evicts expired entries without reading their keys`() { - var now = 0L - val cache = SabrPreparedInfoCache( - ttl = Duration.ofMillis(10), - clock = { now }, - ) - cache.put("video", 0L, preparedInfo(listOf(format(true), format(false)))) - now = 10L - - cache.evictExpired() - - assertFalse(cache.get("video", 0L)?.hasAudioAndVideoFormats() == true) - } - - @Test - fun `cache limits retained preparation windows`() { - val cache = SabrPreparedInfoCache(maxEntries = 1) - val prepared = preparedInfo(listOf(format(true), format(false))) - cache.put("first", 0L, prepared) - - cache.put("second", 0L, prepared) - - assertFalse(cache.get("first", 0L)?.hasAudioAndVideoFormats() == true) - assertSame(prepared, cache.get("second", 0L)) - } - - @Test - fun `info fetcher keeps extracted sabr info for initialization fallback`() = runTest { - val audio = format(isAudio = true) - val video = format(isAudio = false) - val info = info(listOf(audio, video)) - val fetcher = SabrInfoFetcher(mockk(relaxed = true)) - - fetcher.rememberExtractedInfo("video", info) - - assertSame(video, fetcher.initializationFormat("video", video)) - } - - @Test - fun `info fetcher consumes token session metadata and tokens atomically`() = runTest { - val tokenClient = mockk() - val sessionClient = mockk() - val info = info(listOf(format(isAudio = true), format(isAudio = false))) - val token = token() - coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, token) - val fetcher = SabrInfoFetcher(tokenClient, sessionClient) - - val result = fetcher.fetchInfo("video") - - assertEquals(info.formats, result?.info?.formats) - assertSame(token, result?.initialToken) - coVerify(exactly = 1) { sessionClient.fetchPlaybackSession("video") } - verify(exactly = 0) { tokenClient.fetch(any(), any(), any()) } - } - - @Test - fun `info fetcher accepts legacy token session only with matching visitor data`() = runTest { - val tokenClient = mockk() - val sessionClient = mockk() - val info = info(listOf(format(isAudio = true), format(isAudio = false))) - val token = token() - coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, null) - every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns token - val fetcher = SabrInfoFetcher(tokenClient, sessionClient) - - val result = fetcher.fetchInfo("video") - - assertEquals(info.formats, result?.info?.formats) - assertSame(token, result?.initialToken) - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } - } - - @Test - fun `info fetcher rejects mismatched session token generation`() = runTest { - val tokenClient = mockk() - val sessionClient = mockk() - val info = info(listOf(format(isAudio = true), format(isAudio = false))) - val mismatched = token(visitorData = "other-visitor") - val fallback = token() - coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, mismatched) - every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns fallback - val fetcher = SabrInfoFetcher(tokenClient, sessionClient) - - val result = fetcher.fetchInfo("video") - - assertEquals(info.formats, result?.info?.formats) - assertSame(fallback, result?.initialToken) - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } - } - - @Test - fun `info fetcher keeps refreshed player context and metadata together`() = runTest { - val tokenClient = mockk() - val initial = token(visitorData = "old-visitor") - val refreshed = token(visitorData = "fresh-visitor") - val info = info(listOf(format(isAudio = true), format(isAudio = false)), "fresh-visitor") - every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns initial - every { tokenClient.fetch("video", forceRefresh = true, refreshVideo = false) } returns refreshed - val probe = SabrPlayerInfoProbe { _, profile, token -> - if (token === initial) { - throw dev.typetype.server.sabr.SabrProtocolException( - "Player response has no streamingData for $profile", - ) - } - info - } - val fetcher = SabrInfoFetcher(tokenClient, playerInfoProbe = probe) - - val result = fetcher.fetchInfo("video") - - assertSame(info, result?.info) - assertSame(refreshed, result?.initialToken) - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true, refreshVideo = false) } - } - - private fun preparedInfo(formats: List): SabrPreparedInfo { - return SabrPreparedInfo(info(formats), token()) - } - - private fun info( - formats: List, - visitorData: String = "visitor-data", - ): YoutubeSabrInfo { - val info = mockk() - every { info.formats } returns formats - every { info.visitorData } returns visitorData - every { info.profile } returns YoutubeSabrClientProfile.MWEB - every { info.videoId } returns "video" - every { info.cpn } returns "cpn" - every { info.clientVersion } returns "2.20260718.00.00" - every { info.serverAbrStreamingUrl } returns "https://example.com/sabr" - every { info.videoPlaybackUstreamerConfig } returns "config" - return info - } - - private fun tokenSession(info: YoutubeSabrInfo, token: SabrTokenBundle?): TokenYoutubeSession = TokenYoutubeSession( - info = info, - token = token, - title = "", - author = "", - channelId = "", - channelAvatarUrl = "", - description = "", - durationMs = 0L, - viewCount = 0L, - thumbnailUrl = "", - tags = emptyList(), - isLive = false, - isLiveContent = false, - ) - - private fun format(isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.isAudio } returns isAudio - every { format.isVideo } returns !isAudio - every { format.itag } returns if (isAudio) 140 else 137 - every { format.audioTrackId } returns null - every { format.xtags } returns null - return format - } - - private fun token(visitorData: String = "visitor-data"): SabrTokenBundle = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = "visitor", - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video", - videoBoundPoTokenBytes = byteArrayOf(2), - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt index f4981740..2ce9c2a1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt @@ -1,47 +1,229 @@ package dev.typetype.server.services -private const val DEFAULT_SABR_PROBE_VIDEO_ID = "MO7hCeL-zRU" -private const val DEFAULT_SABR_PROBE_PLAYER_TIME_MS = 321_601L -private const val DEFAULT_SABR_PROBE_VIDEO_ITAG = 137 -private const val DEFAULT_SABR_PROBE_AUDIO_ITAG = 140 -private const val DEFAULT_SABR_PROBE_TIMEOUT_MS = 60_000L - -internal fun sabrProbeTokenServiceUrl(): String = - envValue("SUBTITLE_SERVICE_URL") ?: "http://localhost:8081" - -internal fun sabrProbeVideoId(): String = - envValue("SABR_PROBE_VIDEO") - ?: envValues("SABR_PROBE_VIDEOS").firstOrNull() - ?: DEFAULT_SABR_PROBE_VIDEO_ID - -internal fun sabrProbeVideoIds(): List = - envValues("SABR_PROBE_VIDEOS").ifEmpty { listOf(sabrProbeVideoId()) } - -internal fun sabrProbePlayerTimeMs(): Long = - envValue("SABR_PROBE_PLAYER_TIME_MS")?.toLongOrNull()?.takeIf { it >= 0L } - ?: DEFAULT_SABR_PROBE_PLAYER_TIME_MS - -internal fun sabrProbeTimeoutMs(): Long = - envValue("SABR_PROBE_FETCH_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } - ?: envValue("SABR_PROBE_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } - ?: DEFAULT_SABR_PROBE_TIMEOUT_MS - -internal fun sabrProbeAudioItag(): Int = - envValue("SABR_PROBE_AUDIO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_AUDIO_ITAG - -internal fun sabrProbeVideoItags(): List { - val explicit = envIntValues("SABR_PROBE_VIDEO_ITAGS") - val primary = envValue("SABR_PROBE_VIDEO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ITAG - val base = explicit.ifEmpty { listOf(primary) } - val extra720 = envValue("SABR_PROBE_720P_ITAG")?.toIntOrNull() - return (base + listOfNotNull(extra720)).distinct() -} +import io.mockk.every +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import java.time.Duration + +class SabrPreparedInfoCacheTest { + @Test + fun `prepared info reports audio and video only when both are present`() { + assertFalse(preparedInfo(listOf(format(isAudio = false))).hasAudioAndVideoFormats()) + assertFalse(preparedInfo(listOf(format(isAudio = true))).hasAudioAndVideoFormats()) + assertTrue(preparedInfo(listOf(format(isAudio = true), format(isAudio = false))).hasAudioAndVideoFormats()) + } + + @Test + fun `cache can remove incomplete prepared info`() { + val cache = SabrPreparedInfoCache() + cache.put("video", 120_000L, preparedInfo(listOf(format(isAudio = false)))) + + cache.remove("video", 120_000L) + + assertFalse(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) + } + + @Test + fun `cache separates prepared info by start window`() { + val cache = SabrPreparedInfoCache() + cache.put("video", 120_000L, preparedInfo(listOf(format(isAudio = true), format(isAudio = false)))) + + assertTrue(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) + assertFalse(cache.get("video", 340_000L)?.hasAudioAndVideoFormats() == true) + } + + @Test + fun `cache removes every start window for a video`() { + val cache = SabrPreparedInfoCache() + val prepared = preparedInfo(listOf(format(isAudio = true), format(isAudio = false))) + cache.put("video", 120_000L, prepared) + cache.put("video", 340_000L, prepared) + + cache.remove("video") + + assertFalse(cache.get("video", 120_000L)?.hasAudioAndVideoFormats() == true) + assertFalse(cache.get("video", 340_000L)?.hasAudioAndVideoFormats() == true) + } + + @Test + fun `cache evicts expired entries without reading their keys`() { + var now = 0L + val cache = SabrPreparedInfoCache( + ttl = Duration.ofMillis(10), + clock = { now }, + ) + cache.put("video", 0L, preparedInfo(listOf(format(true), format(false)))) + now = 10L + + cache.evictExpired() + + assertFalse(cache.get("video", 0L)?.hasAudioAndVideoFormats() == true) + } + + @Test + fun `cache limits retained preparation windows`() { + val cache = SabrPreparedInfoCache(maxEntries = 1) + val prepared = preparedInfo(listOf(format(true), format(false))) + cache.put("first", 0L, prepared) + + cache.put("second", 0L, prepared) + + assertFalse(cache.get("first", 0L)?.hasAudioAndVideoFormats() == true) + assertSame(prepared, cache.get("second", 0L)) + } + + @Test + fun `info fetcher keeps extracted sabr info for initialization fallback`() = runTest { + val audio = format(isAudio = true) + val video = format(isAudio = false) + val info = info(listOf(audio, video)) + val fetcher = SabrInfoFetcher(mockk(relaxed = true)) -private fun envValue(name: String): String? = - System.getenv(name)?.trim()?.takeIf { it.isNotEmpty() } + fetcher.rememberExtractedInfo("video", info) -private fun envValues(name: String): List = - envValue(name)?.split(",")?.map { it.trim() }?.filter { it.isNotEmpty() }.orEmpty() + assertSame(video, fetcher.initializationFormat("video", video)) + } -private fun envIntValues(name: String): List = - envValues(name).mapNotNull { it.toIntOrNull() } + @Test + fun `info fetcher consumes token session metadata and tokens atomically`() = runTest { + val tokenClient = mockk() + val sessionClient = mockk() + val info = info(listOf(format(isAudio = true), format(isAudio = false))) + val token = token() + coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, token) + val fetcher = SabrInfoFetcher(tokenClient, sessionClient) + + val result = fetcher.fetchInfo("video") + + assertEquals(info.formats, result?.info?.formats) + assertSame(token, result?.initialToken) + coVerify(exactly = 1) { sessionClient.fetchPlaybackSession("video") } + verify(exactly = 0) { tokenClient.fetch(any(), any(), any()) } + } + + @Test + fun `info fetcher accepts legacy token session only with matching visitor data`() = runTest { + val tokenClient = mockk() + val sessionClient = mockk() + val info = info(listOf(format(isAudio = true), format(isAudio = false))) + val token = token() + coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, null) + every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns token + val fetcher = SabrInfoFetcher(tokenClient, sessionClient) + + val result = fetcher.fetchInfo("video") + + assertEquals(info.formats, result?.info?.formats) + assertSame(token, result?.initialToken) + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } + } + + @Test + fun `info fetcher rejects mismatched session token generation`() = runTest { + val tokenClient = mockk() + val sessionClient = mockk() + val info = info(listOf(format(isAudio = true), format(isAudio = false))) + val mismatched = token(visitorData = "other-visitor") + val fallback = token() + coEvery { sessionClient.fetchPlaybackSession("video") } returns tokenSession(info, mismatched) + every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns fallback + val fetcher = SabrInfoFetcher(tokenClient, sessionClient) + + val result = fetcher.fetchInfo("video") + + assertEquals(info.formats, result?.info?.formats) + assertSame(fallback, result?.initialToken) + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } + } + + @Test + fun `info fetcher keeps refreshed player context and metadata together`() = runTest { + val tokenClient = mockk() + val initial = token(visitorData = "old-visitor") + val refreshed = token(visitorData = "fresh-visitor") + val info = info(listOf(format(isAudio = true), format(isAudio = false)), "fresh-visitor") + every { tokenClient.fetch("video", forceRefresh = false, refreshVideo = false) } returns initial + every { tokenClient.fetch("video", forceRefresh = true, refreshVideo = false) } returns refreshed + val probe = SabrPlayerInfoProbe { _, profile, token -> + if (token === initial) { + throw dev.typetype.server.sabr.SabrProtocolException( + "Player response has no streamingData for $profile", + ) + } + info + } + val fetcher = SabrInfoFetcher(tokenClient, playerInfoProbe = probe) + + val result = fetcher.fetchInfo("video") + + assertSame(info, result?.info) + assertSame(refreshed, result?.initialToken) + verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true, refreshVideo = false) } + } + + private fun preparedInfo(formats: List): SabrPreparedInfo { + return SabrPreparedInfo(info(formats), token()) + } + + private fun info( + formats: List, + visitorData: String = "visitor-data", + ): YoutubeSabrInfo { + val info = mockk() + every { info.formats } returns formats + every { info.visitorData } returns visitorData + every { info.profile } returns YoutubeSabrClientProfile.MWEB + every { info.videoId } returns "video" + every { info.cpn } returns "cpn" + every { info.clientVersion } returns "2.20260718.00.00" + every { info.serverAbrStreamingUrl } returns "https://example.com/sabr" + every { info.videoPlaybackUstreamerConfig } returns "config" + return info + } + + private fun tokenSession(info: YoutubeSabrInfo, token: SabrTokenBundle?): TokenYoutubeSession = TokenYoutubeSession( + info = info, + token = token, + title = "", + author = "", + channelId = "", + channelAvatarUrl = "", + description = "", + durationMs = 0L, + viewCount = 0L, + thumbnailUrl = "", + tags = emptyList(), + isLive = false, + isLiveContent = false, + ) + + private fun format(isAudio: Boolean): YoutubeSabrFormat { + val format = mockk() + every { format.isAudio } returns isAudio + every { format.isVideo } returns !isAudio + every { format.itag } returns if (isAudio) 140 else 137 + every { format.audioTrackId } returns null + every { format.xtags } returns null + return format + } + + private fun token(visitorData: String = "visitor-data"): SabrTokenBundle = SabrTokenBundle( + videoId = "video", + visitorBoundPoToken = "visitor", + visitorBoundPoTokenBytes = byteArrayOf(1), + visitorData = visitorData, + videoBoundPoToken = "video", + videoBoundPoTokenBytes = byteArrayOf(2), + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt index 7c28b7f1..f4981740 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt @@ -1,63 +1,47 @@ package dev.typetype.server.services -import dev.typetype.server.sabr.SabrMediaSegment -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat - -internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { - println( - "$label format itag=${format.itag} audio=${format.isAudio} video=${format.isVideo} " + - "size=${format.width}x${format.height} bitrate=${format.bitrate} mime=${format.mimeType} " + - "quality=${format.qualityLabel} audioTrack=${format.audioTrackId} " + - "xtags=${format.xtags} drc=${format.isDrc} original=${format.isOriginalAudio} " + - "approxDurationMs=${format.approxDurationMs}" - ) -} +private const val DEFAULT_SABR_PROBE_VIDEO_ID = "MO7hCeL-zRU" +private const val DEFAULT_SABR_PROBE_PLAYER_TIME_MS = 321_601L +private const val DEFAULT_SABR_PROBE_VIDEO_ITAG = 137 +private const val DEFAULT_SABR_PROBE_AUDIO_ITAG = 140 +private const val DEFAULT_SABR_PROBE_TIMEOUT_MS = 60_000L -internal fun printSabrProbeFetch( - label: String, - holder: SabrSessionHolder, - request: SabrSegmentRequest, - result: SabrProbeFetchResult, -): Unit { - val outcome = when { - result.timedOut -> "timeout" - result.error != null -> "error" - result.segment == null -> "miss" - else -> "hit" - } - println( - "$label request ${sabrProbeRequestSummary(holder, request)} " + - "elapsedMs=${result.elapsedMs} outcome=$outcome" - ) - result.segment?.let { println("$label header ${sabrProbeSegmentHeader(it)}") } - result.error?.let { println("$label error ${it.javaClass.simpleName}: ${it.message}") } -} +internal fun sabrProbeTokenServiceUrl(): String = + envValue("SUBTITLE_SERVICE_URL") ?: "http://localhost:8081" -internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { - val header = segment.header - return "headerId=${header.headerId} videoId=${header.videoId} itag=${header.itag} " + - "seq=${header.sequenceNumber} init=${header.isInitSegment} startMs=${header.startMs} " + - "durationMs=${header.durationMs} bitrateBps=${header.bitrateBps} " + - "contentLength=${header.contentLength} bytes=${segment.length} summary=${header.summarize()}" -} +internal fun sabrProbeVideoId(): String = + envValue("SABR_PROBE_VIDEO") + ?: envValues("SABR_PROBE_VIDEOS").firstOrNull() + ?: DEFAULT_SABR_PROBE_VIDEO_ID + +internal fun sabrProbeVideoIds(): List = + envValues("SABR_PROBE_VIDEOS").ifEmpty { listOf(sabrProbeVideoId()) } + +internal fun sabrProbePlayerTimeMs(): Long = + envValue("SABR_PROBE_PLAYER_TIME_MS")?.toLongOrNull()?.takeIf { it >= 0L } + ?: DEFAULT_SABR_PROBE_PLAYER_TIME_MS -internal fun sabrProbeRequestSummary( - holder: SabrSessionHolder, - request: SabrSegmentRequest, -): String { - val expectedStartMs = if (request.isInitializationSegment) { - -1L - } else { - holder.session.streamState.getSegmentStartMs(request.format, request.sequenceNumber) - } - val expectedEndMs = if (request.isInitializationSegment) { - -1L - } else { - holder.session.streamState.getSegmentEndMs(request.format, request.sequenceNumber) - } - return "itag=${request.format.itag} seq=${request.sequenceNumber} init=${request.isInitializationSegment} " + - "expectedStartMs=$expectedStartMs expectedEndMs=$expectedEndMs " + - "edgeMs=${holder.session.streamState.getMinBufferedEndMs()} " + - "requestNumber=${holder.session.requestNumber} cachedBytes=${holder.session.cachedBytes}" +internal fun sabrProbeTimeoutMs(): Long = + envValue("SABR_PROBE_FETCH_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } + ?: envValue("SABR_PROBE_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } + ?: DEFAULT_SABR_PROBE_TIMEOUT_MS + +internal fun sabrProbeAudioItag(): Int = + envValue("SABR_PROBE_AUDIO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_AUDIO_ITAG + +internal fun sabrProbeVideoItags(): List { + val explicit = envIntValues("SABR_PROBE_VIDEO_ITAGS") + val primary = envValue("SABR_PROBE_VIDEO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ITAG + val base = explicit.ifEmpty { listOf(primary) } + val extra720 = envValue("SABR_PROBE_720P_ITAG")?.toIntOrNull() + return (base + listOfNotNull(extra720)).distinct() } + +private fun envValue(name: String): String? = + System.getenv(name)?.trim()?.takeIf { it.isNotEmpty() } + +private fun envValues(name: String): List = + envValue(name)?.split(",")?.map { it.trim() }?.filter { it.isNotEmpty() }.orEmpty() + +private fun envIntValues(name: String): List = + envValues(name).mapNotNull { it.toIntOrNull() } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt index 86f2cf76..7c28b7f1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt @@ -1,41 +1,63 @@ package dev.typetype.server.services -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat -internal suspend fun fetchSabrProbeSegment( - store: SabrSessionStore, +internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { + println( + "$label format itag=${format.itag} audio=${format.isAudio} video=${format.isVideo} " + + "size=${format.width}x${format.height} bitrate=${format.bitrate} mime=${format.mimeType} " + + "quality=${format.qualityLabel} audioTrack=${format.audioTrackId} " + + "xtags=${format.xtags} drc=${format.isDrc} original=${format.isOriginalAudio} " + + "approxDurationMs=${format.approxDurationMs}" + ) +} + +internal fun printSabrProbeFetch( + label: String, holder: SabrSessionHolder, request: SabrSegmentRequest, - timeoutMs: Long, -): SabrProbeFetchResult { - val startedAt = System.nanoTime() - val result = withTimeoutOrNull(timeoutMs) { - try { - Result.success(store.fetchSegment(holder, request)) - } catch (error: CancellationException) { - throw error - } catch (error: Throwable) { - Result.failure(error) - } + result: SabrProbeFetchResult, +): Unit { + val outcome = when { + result.timedOut -> "timeout" + result.error != null -> "error" + result.segment == null -> "miss" + else -> "hit" } - val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L - if (result == null) { - return SabrProbeFetchResult( - segment = null, - elapsedMs = elapsedMs, - timedOut = true, - error = null, - ) - } - return result.fold( - onSuccess = { - SabrProbeFetchResult(segment = it, elapsedMs = elapsedMs, timedOut = false, error = null) - }, - onFailure = { - SabrProbeFetchResult(segment = null, elapsedMs = elapsedMs, timedOut = false, error = it) - }, + println( + "$label request ${sabrProbeRequestSummary(holder, request)} " + + "elapsedMs=${result.elapsedMs} outcome=$outcome" ) + result.segment?.let { println("$label header ${sabrProbeSegmentHeader(it)}") } + result.error?.let { println("$label error ${it.javaClass.simpleName}: ${it.message}") } +} + +internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { + val header = segment.header + return "headerId=${header.headerId} videoId=${header.videoId} itag=${header.itag} " + + "seq=${header.sequenceNumber} init=${header.isInitSegment} startMs=${header.startMs} " + + "durationMs=${header.durationMs} bitrateBps=${header.bitrateBps} " + + "contentLength=${header.contentLength} bytes=${segment.length} summary=${header.summarize()}" +} + +internal fun sabrProbeRequestSummary( + holder: SabrSessionHolder, + request: SabrSegmentRequest, +): String { + val expectedStartMs = if (request.isInitializationSegment) { + -1L + } else { + holder.session.streamState.getSegmentStartMs(request.format, request.sequenceNumber) + } + val expectedEndMs = if (request.isInitializationSegment) { + -1L + } else { + holder.session.streamState.getSegmentEndMs(request.format, request.sequenceNumber) + } + return "itag=${request.format.itag} seq=${request.sequenceNumber} init=${request.isInitializationSegment} " + + "expectedStartMs=$expectedStartMs expectedEndMs=$expectedEndMs " + + "edgeMs=${holder.session.streamState.getMinBufferedEndMs()} " + + "requestNumber=${holder.session.requestNumber} cachedBytes=${holder.session.cachedBytes}" } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt index 55d799d9..86f2cf76 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt @@ -1,10 +1,41 @@ package dev.typetype.server.services +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest -internal data class SabrProbeFetchResult( - val segment: SabrMediaSegment?, - val elapsedMs: Long, - val timedOut: Boolean, - val error: Throwable?, -) +internal suspend fun fetchSabrProbeSegment( + store: SabrSessionStore, + holder: SabrSessionHolder, + request: SabrSegmentRequest, + timeoutMs: Long, +): SabrProbeFetchResult { + val startedAt = System.nanoTime() + val result = withTimeoutOrNull(timeoutMs) { + try { + Result.success(store.fetchSegment(holder, request)) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + Result.failure(error) + } + } + val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L + if (result == null) { + return SabrProbeFetchResult( + segment = null, + elapsedMs = elapsedMs, + timedOut = true, + error = null, + ) + } + return result.fold( + onSuccess = { + SabrProbeFetchResult(segment = it, elapsedMs = elapsedMs, timedOut = false, error = null) + }, + onFailure = { + SabrProbeFetchResult(segment = null, elapsedMs = elapsedMs, timedOut = false, error = it) + }, + ) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt index 9b764213..55d799d9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt @@ -1,93 +1,10 @@ package dev.typetype.server.services -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization -import dev.typetype.server.sabr.YoutubeSabrClientProfile -import dev.typetype.server.sabr.SabrAdapter -import dev.typetype.server.sabr.YoutubeSabrSession - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrProbeTest { - - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() - - private val probeVideos: List = - sabrProbeVideoIds() - - @Test - fun probeSabr(): Unit { - NewPipeInitializer.init() - val loc = Localization("en", "US") - val country = ContentCountry("US") - val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) - val profile = YoutubeSabrClientProfile.WEB - val playerTimeMs = sabrProbePlayerTimeMs() - val audioItag = sabrProbeAudioItag() - val videoItags = sabrProbeVideoItags() - println( - "config videos=${probeVideos.joinToString(",")} playerTimeMs=$playerTimeMs " + - "audioItag=$audioItag videoItags=${videoItags.joinToString(",")}" - ) - - for (videoId in probeVideos) { - println("\n========== SABR probe: $videoId ==========") - try { - val token = tokenClient.fetch(videoId) ?: error("No SABR token") - val info = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { - SabrAdapter.fetchSabrInfo(videoId, profile, loc, country) - } - println("serverAbrStreamingUrl present: ${!info.serverAbrStreamingUrl.isNullOrEmpty()}") - println("videoPlaybackUstreamerConfig present: ${!info.videoPlaybackUstreamerConfig.isNullOrEmpty()}") - println("--- formats (itag | A/V | height×width | bitrate | mime | audioTrack | approxDurMs) ---") - info.formats.forEach { f -> - val kind = when { - f.isAudio -> "A" - f.isVideo -> "V" - else -> "?" - } - println( - " ${f.itag} | $kind | ${f.height}×${f.width} | br=${f.bitrate} | " + - "${f.mimeType} | track=${f.audioTrackId} | dur≈${f.approxDurationMs}ms" - ) - } - val hasAvc = info.formats.any { it.itag == 137 } - println("AVC itag 137 present: $hasAvc") - for (videoItag in videoItags) { - val configuredVideo = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } - println("configured video itag $videoItag present: ${configuredVideo != null}") - configuredVideo?.let { printSabrProbeFormat("configured video[$videoItag]", it) } - } - - val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } - ?: info.findBestAudioFormat() - val video = info.formats.firstOrNull { it.itag == videoItags.first() && it.isVideo } - ?: info.findLowestVideoFormat() - println("selected pair: audio=${audio?.itag} video=${video?.itag}") - if (audio == null || video == null) { - println("no usable audio/video pair — skipping session") - continue - } - - val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, token) - val session = YoutubeSabrSession(info, audio, video, provider) - session.streamState.setActiveTrackTypes(true, true) - session.streamState.setPlayerTimeMs(playerTimeMs) - println("--- pumpOnce #1 ---") - val segments = session.pumpOnce(loc) - println("segments returned: ${segments.size}") - segments.take(8).forEach { s -> - println(" ${sabrProbeSegmentHeader(s)}") - } - println("session complete: ${session.isComplete}") - } catch (e: Exception) { - println("FAILED on $videoId: ${e.javaClass.simpleName}: ${e.message}") - e.printStackTrace() - } - } - } -} +import dev.typetype.server.sabr.SabrMediaSegment + +internal data class SabrProbeFetchResult( + val segment: SabrMediaSegment?, + val elapsedMs: Long, + val timedOut: Boolean, + val error: Throwable?, +) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt index 0552eddf..9b764213 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProtectedContextRecoveryTest.kt @@ -1,49 +1,93 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import io.mockk.verify +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization +import dev.typetype.server.sabr.YoutubeSabrClientProfile +import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.sabr.YoutubeSabrSession -class SabrProtectedContextRecoveryTest { - @Test - fun `refreshes the active rejected context`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns token("rejected") - every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrProbeTest { - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } - } + private val probeVideos: List = + sabrProbeVideoIds() @Test - fun `keeps a context already refreshed by another recovery`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns token("fresh") + fun probeSabr(): Unit { + NewPipeInitializer.init() + val loc = Localization("en", "US") + val country = ContentCountry("US") + val tokenClient = TypetypeTokenSabrTokenClient(tokenServiceUrl) + val profile = YoutubeSabrClientProfile.WEB + val playerTimeMs = sabrProbePlayerTimeMs() + val audioItag = sabrProbeAudioItag() + val videoItags = sabrProbeVideoItags() + println( + "config videos=${probeVideos.joinToString(",")} playerTimeMs=$playerTimeMs " + + "audioItag=$audioItag videoItags=${videoItags.joinToString(",")}" + ) - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") + for (videoId in probeVideos) { + println("\n========== SABR probe: $videoId ==========") + try { + val token = tokenClient.fetch(videoId) ?: error("No SABR token") + val info = TypetypeYoutubeSessionPoTokenProvider.withToken(token) { + SabrAdapter.fetchSabrInfo(videoId, profile, loc, country) + } + println("serverAbrStreamingUrl present: ${!info.serverAbrStreamingUrl.isNullOrEmpty()}") + println("videoPlaybackUstreamerConfig present: ${!info.videoPlaybackUstreamerConfig.isNullOrEmpty()}") + println("--- formats (itag | A/V | height×width | bitrate | mime | audioTrack | approxDurMs) ---") + info.formats.forEach { f -> + val kind = when { + f.isAudio -> "A" + f.isVideo -> "V" + else -> "?" + } + println( + " ${f.itag} | $kind | ${f.height}×${f.width} | br=${f.bitrate} | " + + "${f.mimeType} | track=${f.audioTrackId} | dur≈${f.approxDurationMs}ms" + ) + } + val hasAvc = info.formats.any { it.itag == 137 } + println("AVC itag 137 present: $hasAvc") + for (videoItag in videoItags) { + val configuredVideo = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } + println("configured video itag $videoItag present: ${configuredVideo != null}") + configuredVideo?.let { printSabrProbeFormat("configured video[$videoItag]", it) } + } - verify(exactly = 0) { tokenClient.fetch("video", forceRefresh = true) } - } + val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } + ?: info.findBestAudioFormat() + val video = info.formats.firstOrNull { it.itag == videoItags.first() && it.isVideo } + ?: info.findLowestVideoFormat() + println("selected pair: audio=${audio?.itag} video=${video?.itag}") + if (audio == null || video == null) { + println("no usable audio/video pair — skipping session") + continue + } - @Test - fun `refreshes when the current context cannot be read`() { - val tokenClient = mockk() - every { tokenClient.fetch("video") } returns null - every { tokenClient.fetch("video", forceRefresh = true) } returns token("fresh") - - SabrProtectedContextRecovery(tokenClient).refreshIfRejected("video", "rejected") - - verify(exactly = 1) { tokenClient.fetch("video", forceRefresh = true) } + val provider = TypetypeTokenSabrPoTokenProvider(tokenClient, token) + val session = YoutubeSabrSession(info, audio, video, provider) + session.streamState.setActiveTrackTypes(true, true) + session.streamState.setPlayerTimeMs(playerTimeMs) + println("--- pumpOnce #1 ---") + val segments = session.pumpOnce(loc) + println("segments returned: ${segments.size}") + segments.take(8).forEach { s -> + println(" ${sabrProbeSegmentHeader(s)}") + } + println("session complete: ${session.isComplete}") + } catch (e: Exception) { + println("FAILED on $videoId: ${e.javaClass.simpleName}: ${e.message}") + e.printStackTrace() + } + } } - - private fun token(visitorData: String): SabrTokenBundle = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = "player-$visitorData", - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video-$visitorData", - videoBoundPoTokenBytes = byteArrayOf(2), - ) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt deleted file mode 100644 index b0440c56..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTest.kt +++ /dev/null @@ -1,139 +0,0 @@ -package dev.typetype.server.services - -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrNextRequestPolicy -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState - -class SabrPumpRuntimeTest { - @Test - fun `startup and seek cushions follow PipePipe policy`() { - var now = 1_000L - val holder = holder(policy(targetAudioMs = 4_000, targetVideoMs = 7_000), playerTimeMs = 1_000L) - val runtime = SabrPumpRuntime { now } - - assertEquals(25_000L, runtime.targetReadaheadCushionMs(holder)) - now += 25_001L - assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) - - runtime.activateSeekMode() - assertEquals(5_000L, runtime.targetReadaheadCushionMs(holder)) - now += 8_000L - assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) - } - - @Test - fun `readahead cushions scale with playback rate while remaining bounded`() { - var now = 1_000L - val holder = holder( - policy(targetAudioMs = 4_000, targetVideoMs = 7_000), - playerTimeMs = 1_000L, - playbackRate = 4.0f, - ) - val runtime = SabrPumpRuntime { now } - - assertEquals(60_000L, runtime.targetReadaheadCushionMs(holder)) - now += 25_001L - assertEquals(28_000L, runtime.targetReadaheadCushionMs(holder)) - runtime.activateSeekMode() - assertEquals(20_000L, runtime.targetReadaheadCushionMs(holder)) - } - - @Test - fun `server heartbeat bypasses time throttling`() { - var now = 1_000L - val holder = holder( - policy(targetAudioMs = 3_000, targetVideoMs = 3_000, maximumRequestGapMs = 5_000), - playerTimeMs = 1_000L, - edgeMs = 13_000L, - ) - val runtime = SabrPumpRuntime { now } - now += 25_001L - - assertTrue(runtime.isThrottled(holder)) - runtime.recordRequest() - now += 4_999L - assertTrue(runtime.isThrottled(holder)) - now += 1L - assertFalse(runtime.isThrottled(holder)) - } - - @Test - fun `startup request caps reported server ahead`() { - var now = 1_000L - val holder = holder(policy(), playerTimeMs = 1_000L) - val runtime = SabrPumpRuntime { now } - - assertEquals(34_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) - now += 25_000L - assertEquals(1_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) - assertEquals(34_000L, runtime.demandPlayerTimeMs(holder, edgeMs = 50_000L)) - } - - @Test - fun `deferred request stays recoverable for watchdog`() { - var now = 1_000L - val runtime = SabrPumpRuntime { now } - runtime.beginDemand("140:44") - now += SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS - - assertEquals( - SabrDemandRecoveryAction.WAIT, - runtime.demandRecoveryAction("140:44", requestPerformed = false, resolved = false), - ) - } - - @Test - fun `response without demanded segment is readvertised once`() { - val runtime = SabrPumpRuntime { 1_000L } - runtime.beginDemand("140:44") - - assertEquals( - SabrDemandRecoveryAction.READVERTISE_TRACK, - runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), - ) - repeat(5) { - assertEquals( - SabrDemandRecoveryAction.WAIT, - runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), - ) - } - } - - private fun holder( - policy: SabrNextRequestPolicy, - playerTimeMs: Long, - edgeMs: Long = 0L, - playbackRate: Float = 1.0f, - ): SabrSessionHolder { - val holder = mockk() - val session = mockk() - val state = mockk() - every { holder.session } returns session - every { holder.playerTimeMs() } returns playerTimeMs - every { holder.playbackRate() } returns playbackRate - every { holder.readerTailMs() } returns 1L - every { session.streamState } returns state - every { session.cachedBytes } returns 0L - every { state.getMinBufferedEndMs() } returns edgeMs - every { state.nextRequestPolicy } returns policy - return holder - } - - private fun policy( - targetAudioMs: Int = -1, - targetVideoMs: Int = -1, - maximumRequestGapMs: Int = -1, - ): SabrNextRequestPolicy { - val policy = mockk() - every { policy.targetAudioReadaheadMs } returns targetAudioMs - every { policy.targetVideoReadaheadMs } returns targetVideoMs - every { policy.maxTimeSinceLastRequestMs } returns maximumRequestGapMs - return policy - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt index 3b058d58..b0440c56 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt @@ -1,159 +1,139 @@ package dev.typetype.server.services -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeoutOrNull +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import org.schabi.newpipe.extractor.NewPipe -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import org.schabi.newpipe.extractor.stream.StreamInfo - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrRandomAccessProbeTest { - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() +import dev.typetype.server.sabr.SabrNextRequestPolicy +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +class SabrPumpRuntimeTest { @Test - fun `fetches post seek stateful responses`(): Unit = runBlocking { - NewPipeInitializer.init() - val videoId = sabrProbeVideoId() - val videoItags = sabrProbeVideoItags() - val audioItag = sabrProbeAudioItag() - val playerTimeMs = sabrProbePlayerTimeMs() - val timeoutMs = sabrProbeTimeoutMs() - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - try { - println( - "config videoId=$videoId playerTimeMs=$playerTimeMs audioItag=$audioItag " + - "videoItags=${videoItags.joinToString(",")} timeoutMs=$timeoutMs " + - "contract=stateful-pump" - ) - if (System.getenv("SABR_PROBE_EXTRACT_FIRST") != "false") { - store.rememberExtractedInfo(videoId, extractSabrInfo(videoId)) - } - val prepared = store.fetchInfo(videoId, playerTimeMs, cachedFirst = true) ?: error("SABR probe failed") - val info = prepared.info - val audio = requireAudioFormat(info.formats, audioItag) - printSabrProbeFormat("audio", audio) - for (videoItag in videoItags) { - val video = requireVideoFormat(info.formats, videoItag) - printSabrProbeFormat("video", video) - val startedAt = System.nanoTime() - val preparation = SabrPlaybackSessionService(store).prepare( - videoId = videoId, - userId = "sabr-random-access-video-$videoItag", - prepared = prepared, - audio = audio, - video = video, - startTimeMs = playerTimeMs, - ) - val holder = preparation.holder - assertInitData(store, holder, audio) - assertInitData(store, holder, video) - holder.setActiveTracks(videoActive = true, audioActive = true) - holder.setPlayerTimeMs(playerTimeMs) - val requests = listOf(audio, video).map { format -> - SabrSegmentRequest.media(format, holder.playbackStartSequence(format, playerTimeMs)) - } - requests.forEach { request -> - println("pump target[$videoItag] ${sabrProbeRequestSummary(holder, request)}") - } - val segments = withTimeoutOrNull(timeoutMs) { - requests.map { request -> - store.requestSegmentDemand(holder, request, holder.activeGeneration()) - awaitCachedSegment(store, holder, request) - } - } - val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L - println("pump[$videoItag] elapsedMs=$elapsedMs segments=${segments?.size ?: -1}") - println("pump[$videoItag] trace=${holder.session.diagnosticTrace}") - segments.orEmpty().forEach { - println( - "pump[$videoItag] itag=${it.itag} seq=${it.sequence} startMs=${it.startMs} " + - "durationMs=${it.durationMs} bytes=${it.length}" - ) - } - assertTrue( - segments.orEmpty().any { it.covers(video, playerTimeMs) }, - "video[$videoItag] pump bytes", - ) - assertTrue( - segments.orEmpty().any { it.covers(audio, playerTimeMs) }, - "audio[$videoItag] pump bytes", - ) - } - } finally { - store.release() - } + fun `startup and seek cushions follow PipePipe policy`() { + var now = 1_000L + val holder = holder(policy(targetAudioMs = 4_000, targetVideoMs = 7_000), playerTimeMs = 1_000L) + val runtime = SabrPumpRuntime { now } + + assertEquals(25_000L, runtime.targetReadaheadCushionMs(holder)) + now += 25_001L + assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) + + runtime.activateSeekMode() + assertEquals(5_000L, runtime.targetReadaheadCushionMs(holder)) + now += 8_000L + assertEquals(7_000L, runtime.targetReadaheadCushionMs(holder)) } - private suspend fun awaitCachedSegment( - store: SabrSessionStore, - holder: SabrSessionHolder, - request: SabrSegmentRequest, - ): CachedSabrSegment { - var segment = store.cachedSegment(holder, request) - while (segment == null) { - delay(50L) - segment = store.cachedSegment(holder, request) - } - return segment + @Test + fun `readahead cushions scale with playback rate while remaining bounded`() { + var now = 1_000L + val holder = holder( + policy(targetAudioMs = 4_000, targetVideoMs = 7_000), + playerTimeMs = 1_000L, + playbackRate = 4.0f, + ) + val runtime = SabrPumpRuntime { now } + + assertEquals(60_000L, runtime.targetReadaheadCushionMs(holder)) + now += 25_001L + assertEquals(28_000L, runtime.targetReadaheadCushionMs(holder)) + runtime.activateSeekMode() + assertEquals(20_000L, runtime.targetReadaheadCushionMs(holder)) + } + + @Test + fun `server heartbeat bypasses time throttling`() { + var now = 1_000L + val holder = holder( + policy(targetAudioMs = 3_000, targetVideoMs = 3_000, maximumRequestGapMs = 5_000), + playerTimeMs = 1_000L, + edgeMs = 13_000L, + ) + val runtime = SabrPumpRuntime { now } + now += 25_001L + + assertTrue(runtime.isThrottled(holder)) + runtime.recordRequest() + now += 4_999L + assertTrue(runtime.isThrottled(holder)) + now += 1L + assertFalse(runtime.isThrottled(holder)) } - private suspend fun assertInitData( - store: SabrSessionStore, - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - ): Unit { - val data = store.fetchInitializationData(holder, format) - println("init itag=${format.itag} bytes=${data?.size ?: -1}") - if (data == null || data.isEmpty()) println("init trace=${holder.session.diagnosticTrace}") - assertTrue(data?.isNotEmpty() == true, "init bytes for itag ${format.itag}") + @Test + fun `startup request caps reported server ahead`() { + var now = 1_000L + val holder = holder(policy(), playerTimeMs = 1_000L) + val runtime = SabrPumpRuntime { now } + + assertEquals(34_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) + now += 25_000L + assertEquals(1_000L, runtime.requestPlayerTimeMs(holder, edgeMs = 50_000L)) + assertEquals(34_000L, runtime.demandPlayerTimeMs(holder, edgeMs = 50_000L)) + } + + @Test + fun `deferred request stays recoverable for watchdog`() { + var now = 1_000L + val runtime = SabrPumpRuntime { now } + runtime.beginDemand("140:44") + now += SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS + + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = false, resolved = false), + ) + } + + @Test + fun `response without demanded segment is readvertised once`() { + val runtime = SabrPumpRuntime { 1_000L } + runtime.beginDemand("140:44") + + assertEquals( + SabrDemandRecoveryAction.READVERTISE_TRACK, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) + repeat(5) { + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("140:44", requestPerformed = true, resolved = false), + ) + } } - private fun requireAudioFormat( - formats: List, - audioItag: Int, - ): YoutubeSabrFormat = - formats.filter { it.itag == audioItag && it.isAudio } - .maxWithOrNull(compareBy { it.isOriginalAudio } - .thenBy { it.xtags.isNullOrBlank() } - .thenBy { !it.isDrc } - .thenBy { it.bitrate }) - ?: error("No SABR audio format for itag $audioItag") - - private fun extractSabrInfo(videoId: String): YoutubeSabrInfo { - val service = NewPipe.getServiceByUrl("https://www.youtube.com/watch?v=$videoId") - val linkHandler = service.streamLHFactory.fromUrl("https://www.youtube.com/watch?v=$videoId") - val extractor = service.getStreamExtractor(linkHandler) - extractor.fetchPage() - val streamInfo = StreamInfo.getInfo(extractor) - return sequence { - streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } - streamInfo.videoOnlyStreams.forEach { yield(it.deliveryMethodInfo) } - streamInfo.audioStreams.forEach { yield(it.deliveryMethodInfo) } - }.filterIsInstance() - .first { it.videoId == videoId } + private fun holder( + policy: SabrNextRequestPolicy, + playerTimeMs: Long, + edgeMs: Long = 0L, + playbackRate: Float = 1.0f, + ): SabrSessionHolder { + val holder = mockk() + val session = mockk() + val state = mockk() + every { holder.session } returns session + every { holder.playerTimeMs() } returns playerTimeMs + every { holder.playbackRate() } returns playbackRate + every { holder.readerTailMs() } returns 1L + every { session.streamState } returns state + every { session.cachedBytes } returns 0L + every { state.getMinBufferedEndMs() } returns edgeMs + every { state.nextRequestPolicy } returns policy + return holder } - private fun requireVideoFormat( - formats: List, - videoItag: Int, - ): YoutubeSabrFormat = - formats.firstOrNull { it.itag == videoItag && it.isVideo } - ?: error("No SABR video format for itag $videoItag") - - private fun CachedSabrSegment.covers(format: YoutubeSabrFormat, playerTimeMs: Long): Boolean { - if (length <= 0 || init || itag != format.itag) return false - val startMs = this.startMs - val durationMs = this.durationMs - return startMs >= 0 && durationMs > 0 && - playerTimeMs >= startMs && playerTimeMs < startMs + durationMs + private fun policy( + targetAudioMs: Int = -1, + targetVideoMs: Int = -1, + maximumRequestGapMs: Int = -1, + ): SabrNextRequestPolicy { + val policy = mockk() + every { policy.targetAudioReadaheadMs } returns targetAudioMs + every { policy.targetVideoReadaheadMs } returns targetVideoMs + every { policy.maxTimeSinceLastRequestMs } returns maximumRequestGapMs + return policy } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt index a477f8dc..3b058d58 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRecoverablePumpFailureTest.kt @@ -1,67 +1,159 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrRecoverableException +import org.junit.jupiter.api.condition.EnabledIfSystemProperty +import org.schabi.newpipe.extractor.NewPipe +import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant +import org.schabi.newpipe.extractor.stream.StreamInfo -@OptIn(ExperimentalCoroutinesApi::class) -class SabrRecoverablePumpFailureTest { - @Test - fun `recoverable media failure requests a fresh session`() = runTest { - val holder = pumpFailure("Unexpected EOF") +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrRandomAccessProbeTest { + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertTrue(holder.terminalFailure().orEmpty().startsWith(SABR_RECOVERABLE_FAILURE_PREFIX)) + @Test + fun `fetches post seek stateful responses`(): Unit = runBlocking { + NewPipeInitializer.init() + val videoId = sabrProbeVideoId() + val videoItags = sabrProbeVideoItags() + val audioItag = sabrProbeAudioItag() + val playerTimeMs = sabrProbePlayerTimeMs() + val timeoutMs = sabrProbeTimeoutMs() + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + try { + println( + "config videoId=$videoId playerTimeMs=$playerTimeMs audioItag=$audioItag " + + "videoItags=${videoItags.joinToString(",")} timeoutMs=$timeoutMs " + + "contract=stateful-pump" + ) + if (System.getenv("SABR_PROBE_EXTRACT_FIRST") != "false") { + store.rememberExtractedInfo(videoId, extractSabrInfo(videoId)) + } + val prepared = store.fetchInfo(videoId, playerTimeMs, cachedFirst = true) ?: error("SABR probe failed") + val info = prepared.info + val audio = requireAudioFormat(info.formats, audioItag) + printSabrProbeFormat("audio", audio) + for (videoItag in videoItags) { + val video = requireVideoFormat(info.formats, videoItag) + printSabrProbeFormat("video", video) + val startedAt = System.nanoTime() + val preparation = SabrPlaybackSessionService(store).prepare( + videoId = videoId, + userId = "sabr-random-access-video-$videoItag", + prepared = prepared, + audio = audio, + video = video, + startTimeMs = playerTimeMs, + ) + val holder = preparation.holder + assertInitData(store, holder, audio) + assertInitData(store, holder, video) + holder.setActiveTracks(videoActive = true, audioActive = true) + holder.setPlayerTimeMs(playerTimeMs) + val requests = listOf(audio, video).map { format -> + SabrSegmentRequest.media(format, holder.playbackStartSequence(format, playerTimeMs)) + } + requests.forEach { request -> + println("pump target[$videoItag] ${sabrProbeRequestSummary(holder, request)}") + } + val segments = withTimeoutOrNull(timeoutMs) { + requests.map { request -> + store.requestSegmentDemand(holder, request, holder.activeGeneration()) + awaitCachedSegment(store, holder, request) + } + } + val elapsedMs = (System.nanoTime() - startedAt) / 1_000_000L + println("pump[$videoItag] elapsedMs=$elapsedMs segments=${segments?.size ?: -1}") + println("pump[$videoItag] trace=${holder.session.diagnosticTrace}") + segments.orEmpty().forEach { + println( + "pump[$videoItag] itag=${it.itag} seq=${it.sequence} startMs=${it.startMs} " + + "durationMs=${it.durationMs} bytes=${it.length}" + ) + } + assertTrue( + segments.orEmpty().any { it.covers(video, playerTimeMs) }, + "video[$videoItag] pump bytes", + ) + assertTrue( + segments.orEmpty().any { it.covers(audio, playerTimeMs) }, + "audio[$videoItag] pump bytes", + ) + } + } finally { + store.release() + } } - @Test - fun `local spool failure remains terminal without fresh session recovery`() = runTest { - val holder = pumpFailure("Could not write SABR spool file") + private suspend fun awaitCachedSegment( + store: SabrSessionStore, + holder: SabrSessionHolder, + request: SabrSegmentRequest, + ): CachedSabrSegment { + var segment = store.cachedSegment(holder, request) + while (segment == null) { + delay(50L) + segment = store.cachedSegment(holder, request) + } + return segment + } - assertEquals(SabrPlaybackState.TERMINAL, holder.playbackState()) - assertEquals("Could not write SABR spool file", holder.terminalFailure()) + private suspend fun assertInitData( + store: SabrSessionStore, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + ): Unit { + val data = store.fetchInitializationData(holder, format) + println("init itag=${format.itag} bytes=${data?.size ?: -1}") + if (data == null || data.isEmpty()) println("init trace=${holder.session.diagnosticTrace}") + assertTrue(data?.isNotEmpty() == true, "init bytes for itag ${format.itag}") } - private suspend fun pumpFailure(message: String): SabrSessionHolder { - val session = mockk(relaxed = true) - val streamState = mockk(relaxed = true) - every { session.streamState } returns streamState - every { session.requestNumber } returns 1 - every { session.isComplete } returns false - every { session.cachedBytes } returns 0L - every { streamState.nextRequestPolicy } returns null - every { session.pumpOnceStreaming(any()) } throws SabrRecoverableException(message) - val holder = holder(session) + private fun requireAudioFormat( + formats: List, + audioItag: Int, + ): YoutubeSabrFormat = + formats.filter { it.itag == audioItag && it.isAudio } + .maxWithOrNull(compareBy { it.isOriginalAudio } + .thenBy { it.xtags.isNullOrBlank() } + .thenBy { !it.isDrc } + .thenBy { it.bitrate }) + ?: error("No SABR audio format for itag $audioItag") - SabrSessionPump().pumpLoop({ true }, holder, intervalMs = 0L) - return holder + private fun extractSabrInfo(videoId: String): YoutubeSabrInfo { + val service = NewPipe.getServiceByUrl("https://www.youtube.com/watch?v=$videoId") + val linkHandler = service.streamLHFactory.fromUrl("https://www.youtube.com/watch?v=$videoId") + val extractor = service.getStreamExtractor(linkHandler) + extractor.fetchPage() + val streamInfo = StreamInfo.getInfo(extractor) + return sequence { + streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } + streamInfo.videoOnlyStreams.forEach { yield(it.deliveryMethodInfo) } + streamInfo.audioStreams.forEach { yield(it.deliveryMethodInfo) } + }.filterIsInstance() + .first { it.videoId == videoId } } - private fun holder(session: YoutubeSabrSession): SabrSessionHolder = SabrSessionHolder( - session = session, - info = mockk(), - audioFormat = format(140, true), - videoFormat = format(137, false), - sessionToken = "session-token", - key = SabrSessionKey("video", "user", 140, null, 137, 0L), - lastRequestAt = Instant.EPOCH, - ) + private fun requireVideoFormat( + formats: List, + videoItag: Int, + ): YoutubeSabrFormat = + formats.firstOrNull { it.itag == videoItag && it.isVideo } + ?: error("No SABR video format for itag $videoItag") - private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 - return format + private fun CachedSabrSegment.covers(format: YoutubeSabrFormat, playerTimeMs: Long): Boolean { + if (length <= 0 || init || itag != format.itag) return false + val startMs = this.startMs + val durationMs = this.durationMs + return startMs >= 0 && durationMs > 0 && + playerTimeMs >= startMs && playerTimeMs < startMs + durationMs } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt deleted file mode 100644 index 85254a98..00000000 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ /dev/null @@ -1,78 +0,0 @@ -package dev.typetype.server.services - -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.condition.EnabledIfSystemProperty -import dev.typetype.server.sabr.SabrSegmentRequest - -@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") -@Tag("network") -class SabrSessionStoreTest { - - private val tokenServiceUrl: String = - sabrProbeTokenServiceUrl() - - @Test - fun storeRoundTrip(): Unit = runBlocking { - NewPipeInitializer.init() - val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) - val videoId = sabrProbeVideoId() - val playerTimeMs = sabrProbePlayerTimeMs() - val timeoutMs = sabrProbeTimeoutMs() - val audioItag = sabrProbeAudioItag() - val videoItag = sabrProbeVideoItags().first() - val userId = "sabr-probe-user" - - println( - "\n========== SABR store round-trip: $videoId " + - "playerTimeMs=$playerTimeMs timeoutMs=$timeoutMs ==========" - ) - val prepared = store.fetchInfo(videoId) ?: error("SABR probe failed") - val info = prepared.info - val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } - ?: info.findBestAudioFormat() - val video = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } - ?: info.findLowestVideoFormat() - println("probe picked: audio=${audio?.itag} video=${video?.itag}") - check(audio != null && video != null) - printSabrProbeFormat("audio", audio) - printSabrProbeFormat("video", video) - - val holder = store.getOrCreate(videoId, userId, info, audio, video, prepared.initialToken) - store.ensureWarmed(holder) - holder.setActiveTracks(videoActive = true, audioActive = true) - holder.setPlayerTimeMs(playerTimeMs) - println("session complete (cold): ${holder.session.isComplete}") - - val requests = listOf( - SabrSegmentRequest.initialization(video), - SabrSegmentRequest.initialization(audio), - ) + mediaRequestsForProbe(holder, video, audio, playerTimeMs) - for ((i, req) in requests.withIndex()) { - val result = fetchSabrProbeSegment(store, holder, req, timeoutMs) - printSabrProbeFetch("req[$i]", holder, req, result) - } - println("session complete after fetches: ${holder.session.isComplete}") - - val looked = store.lookup(videoId, userId, audio.itag, video.itag) - println("lookup same key: ${looked === holder}") - - store.release() - } - - private fun mediaRequestsForProbe( - holder: SabrSessionHolder, - video: dev.typetype.server.sabr.YoutubeSabrFormat, - audio: dev.typetype.server.sabr.YoutubeSabrFormat, - playerTimeMs: Long, - ): List { - val videoSequence = System.getenv("SABR_PROBE_VIDEO_SEQUENCE")?.toIntOrNull() - val audioSequence = System.getenv("SABR_PROBE_AUDIO_SEQUENCE")?.toIntOrNull() - if (videoSequence == null && audioSequence == null) return holder.mediaRequestsAt(playerTimeMs) - return listOfNotNull( - videoSequence?.let { SabrSegmentRequest.media(video, it) }, - audioSequence?.let { SabrSegmentRequest.media(audio, it) }, - ) - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt index a8b9a821..85254a98 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrSessionTimeRequestsTest.kt @@ -1,189 +1,78 @@ package dev.typetype.server.services -import io.mockk.every -import io.mockk.mockk -import org.junit.jupiter.api.Assertions.assertEquals +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test -import dev.typetype.server.sabr.SabrMediaHeader -import dev.typetype.server.sabr.SabrMediaSegment +import org.junit.jupiter.api.condition.EnabledIfSystemProperty import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat -import dev.typetype.server.sabr.YoutubeSabrInfo -import dev.typetype.server.sabr.YoutubeSabrSession -import dev.typetype.server.sabr.YoutubeSabrStreamState -import java.time.Instant -class SabrSessionTimeRequestsTest { - @Test - fun `playback start sequence is clamped to the known final segment`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 91 - every { state.getEndSegment(audio) } returns 90L - every { state.getEndSegment(video) } returns 0L - val holder = holder(session, audio, video) - - assertEquals(90, holder.playbackStartSequence(audio, 88_168L)) - } - - @Test - fun `playback start sequence stays valid without an indexed end`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 88_168L) } returns 33 - every { state.getEndSegment(audio) } returns 0L - every { state.getEndSegment(video) } returns 0L - val holder = holder(session, audio, video) - - assertEquals(33, holder.playbackStartSequence(audio, 88_168L)) - } - - @Test - fun `mediaRequestsAt returns active audio and video requests for player time`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 321_601L) } returns 33 - val holder = holder(session, audio, video) - - val requests = holder.mediaRequestsAt(321_601L) +@EnabledIfSystemProperty(named = "sabr.probe", matches = "true") +@Tag("network") +class SabrSessionStoreTest { - assertEquals(listOf(137, 140), requests.map { it.format.itag }) - assertEquals(listOf(64, 33), requests.map { it.sequenceNumber }) - } + private val tokenServiceUrl: String = + sabrProbeTokenServiceUrl() @Test - fun `mediaRequestsAt uses mapped video and audio sequences`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 247, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 340_000L) } returns 64 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 340_000L) } returns 35 - val holder = holder(session, audio, video) - - val requests = holder.mediaRequestsAt(340_000L) - - assertEquals(listOf(247, 140), requests.map { it.format.itag }) - assertEquals(listOf(64, 35), requests.map { it.sequenceNumber }) + fun storeRoundTrip(): Unit = runBlocking { + NewPipeInitializer.init() + val store = SabrSessionStore(tokenServiceUrl = tokenServiceUrl) + val videoId = sabrProbeVideoId() + val playerTimeMs = sabrProbePlayerTimeMs() + val timeoutMs = sabrProbeTimeoutMs() + val audioItag = sabrProbeAudioItag() + val videoItag = sabrProbeVideoItags().first() + val userId = "sabr-probe-user" + + println( + "\n========== SABR store round-trip: $videoId " + + "playerTimeMs=$playerTimeMs timeoutMs=$timeoutMs ==========" + ) + val prepared = store.fetchInfo(videoId) ?: error("SABR probe failed") + val info = prepared.info + val audio = info.formats.firstOrNull { it.itag == audioItag && it.isAudio } + ?: info.findBestAudioFormat() + val video = info.formats.firstOrNull { it.itag == videoItag && it.isVideo } + ?: info.findLowestVideoFormat() + println("probe picked: audio=${audio?.itag} video=${video?.itag}") + check(audio != null && video != null) + printSabrProbeFormat("audio", audio) + printSabrProbeFormat("video", video) + + val holder = store.getOrCreate(videoId, userId, info, audio, video, prepared.initialToken) + store.ensureWarmed(holder) + holder.setActiveTracks(videoActive = true, audioActive = true) + holder.setPlayerTimeMs(playerTimeMs) + println("session complete (cold): ${holder.session.isComplete}") + + val requests = listOf( + SabrSegmentRequest.initialization(video), + SabrSegmentRequest.initialization(audio), + ) + mediaRequestsForProbe(holder, video, audio, playerTimeMs) + for ((i, req) in requests.withIndex()) { + val result = fetchSabrProbeSegment(store, holder, req, timeoutMs) + printSabrProbeFetch("req[$i]", holder, req, result) + } + println("session complete after fetches: ${holder.session.isComplete}") + + val looked = store.lookup(videoId, userId, audio.itag, video.itag) + println("lookup same key: ${looked === holder}") + + store.release() } - @Test - fun `mediaRequestsAt excludes inactive tracks`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 137, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentNumberAtOrAfterTimeMs(video, 321_601L) } returns 64 - val holder = holder(session, audio, video) - holder.setActiveTracks(videoActive = true, audioActive = false) - - val requests = holder.mediaRequestsAt(321_601L) - - assertEquals(listOf(137), requests.map { it.format.itag }) - assertEquals(listOf(64), requests.map { it.sequenceNumber }) + private fun mediaRequestsForProbe( + holder: SabrSessionHolder, + video: dev.typetype.server.sabr.YoutubeSabrFormat, + audio: dev.typetype.server.sabr.YoutubeSabrFormat, + playerTimeMs: Long, + ): List { + val videoSequence = System.getenv("SABR_PROBE_VIDEO_SEQUENCE")?.toIntOrNull() + val audioSequence = System.getenv("SABR_PROBE_AUDIO_SEQUENCE")?.toIntOrNull() + if (videoSequence == null && audioSequence == null) return holder.mediaRequestsAt(playerTimeMs) + return listOfNotNull( + videoSequence?.let { SabrSegmentRequest.media(video, it) }, + audioSequence?.let { SabrSegmentRequest.media(audio, it) }, + ) } - - @Test - fun `reposition keeps a non adjacent live boundary request pending`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 248, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { state.getSegmentStartMs(audio, 100) } returns 995_000L - every { session.getCachedSegment(any()) } returns null - val holder = holder(session, audio, video) - val observed = mediaSegment(audio.itag, sequence = 102, startMs = 995_010L) - holder.markExpectedLive() - holder.observeMediaSegment(observed) - every { - session.getCachedSegment(match { - it.format.itag == audio.itag && it.sequenceNumber == 102 - }) - } returns observed - val request = SabrSegmentRequest.media(audio, 100) - - val missing = holder.repositionTargets(listOf(request), playerTimeMs = 995_000L, generation = 0L) - - assertEquals(listOf(request), missing) - } - - @Test - fun `reposition advances to the next warmed live segment from inside a boundary`() { - val audio = sabrFormat(itag = 140, isAudio = true) - val video = sabrFormat(itag = 248, isAudio = false) - val session = mockk() - val state = mockk(relaxed = true) - every { session.streamState } returns state - every { state.setActiveTrackTypes(any(), any()) } returns Unit - every { session.getCachedSegment(any()) } returns null - val holder = holder(session, audio, video) - val observed = mediaSegment(audio.itag, sequence = 101, startMs = 1_000_000L, durationMs = 5_000L) - holder.markExpectedLive() - holder.observeMediaSegment(observed) - every { - session.getCachedSegment(match { - it.format.itag == audio.itag && it.sequenceNumber == 101 - }) - } returns observed - val request = SabrSegmentRequest.media(audio, 100) - - val missing = holder.repositionTargets(listOf(request), playerTimeMs = 996_200L, generation = 0L) - - assertEquals(emptyList(), missing) - assertEquals(1_000_000L, holder.readerPosition(audio)) - } - - private fun sabrFormat(itag: Int, isAudio: Boolean): YoutubeSabrFormat { - val format = mockk() - every { format.itag } returns itag - every { format.isAudio } returns isAudio - every { format.isVideo } returns !isAudio - return format - } - - private fun mediaSegment( - itag: Int, - sequence: Int, - startMs: Long, - durationMs: Long = 0L, - ): SabrMediaSegment { - val header = mockk(relaxed = true) - every { header.itag } returns itag - every { header.sequenceNumber } returns sequence - every { header.startMs } returns startMs - every { header.durationMs } returns durationMs - every { header.isInitSegment } returns false - return mockk { every { this@mockk.header } returns header } - } - - private fun holder( - session: YoutubeSabrSession, - audio: YoutubeSabrFormat, - video: YoutubeSabrFormat, - ): SabrSessionHolder = SabrSessionHolder( - session, - mockk(), - audio, - video, - "session", - SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), - Instant.now(), - ) } diff --git a/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt b/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt deleted file mode 100644 index 53713cf4..00000000 --- a/src/test/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolverTest.kt +++ /dev/null @@ -1,97 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.ExtractionResult -import dev.typetype.server.models.SubtitleItem -import dev.typetype.server.testStreamResponse -import kotlinx.coroutines.runBlocking -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test - -class StreamYouTubeSubtitleResolverTest { - @Test - fun `resolution uses fresh inventory and preserves live state`() = runBlocking { - val resolver = StreamYouTubeSubtitleResolver( - streamService = streamService( - listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=de", false)), - isLive = true, - ), - fetchInventory = { - YouTubeSubtitleInventoryResult.Ready( - listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), - ) - }, - ) - - val result = resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) - - val ready = result as YouTubeSubtitleResolution.Ready - assertTrue(ready.track.isLive) - assertEquals("en", ready.track.content.toHttpUrl().queryParameter("lang")) - } - - @Test - fun `stream inventory is used when fresh inventory is unavailable`() = runBlocking { - val resolver = StreamYouTubeSubtitleResolver( - streamService = streamService( - listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), - ), - fetchInventory = { YouTubeSubtitleInventoryResult.Unavailable }, - ) - - assertTrue(resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) is YouTubeSubtitleResolution.Ready) - } - - @Test - fun `manual and automatic tracks remain distinct`() { - val manual = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false) - val automatic = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr", true) - - assertTrue(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) - assertFalse(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) - assertTrue(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) - assertFalse(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) - } - - @Test - fun `translated selection preserves source language and requested translation`() { - val track = track( - "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr&tlang=de", - true, - ) - val selection = selection(YouTubeSubtitleVariant.Auto).copy( - language = "fr", - sourceLanguage = "en", - translationLanguage = "fr", - ) - - assertTrue(track.matchesYouTubeSubtitle(selection)) - val resolved = requireNotNull(track.contentForYouTubeSubtitle(selection)).toHttpUrl() - assertEquals("en", resolved.queryParameter("lang")) - assertEquals("fr", resolved.queryParameter("tlang")) - assertEquals("vtt", resolved.queryParameter("fmt")) - } - - private fun streamService(subtitles: List, isLive: Boolean = false) = object : StreamService { - override suspend fun getStreamInfo(url: String) = ExtractionResult.Success( - testStreamResponse().copy(subtitles = subtitles, isLive = isLive, isLiveContent = isLive), - ) - } - - private fun track(url: String, automatic: Boolean) = SubtitleItem( - url = url, - mimeType = "application/ttml+xml", - languageTag = "en", - displayLanguageName = "English", - isAutoGenerated = automatic, - ) - - private fun selection(variant: YouTubeSubtitleVariant) = YouTubeSubtitleSelection( - videoId = "abcdefghijk", - language = "en", - variant = variant, - format = YouTubeSubtitleFormat.Vtt, - ) -} diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt index 3a6a12e4..53713cf4 100644 --- a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt @@ -1,96 +1,97 @@ package dev.typetype.server.services +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubtitleItem +import dev.typetype.server.testStreamResponse +import kotlinx.coroutines.runBlocking +import okhttp3.HttpUrl.Companion.toHttpUrl import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider - -class TypetypeYoutubeSessionPoTokenProviderTest { - @AfterEach - fun clearAuthenticatedProvider(): Unit = - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) +class StreamYouTubeSubtitleResolverTest { @Test - fun `exposes the session token only inside its scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - assertEquals("visitor", currentToken()?.visitorData) - assertEquals("player-token", currentToken()?.poToken) - } + fun `resolution uses fresh inventory and preserves live state`() = runBlocking { + val resolver = StreamYouTubeSubtitleResolver( + streamService = streamService( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=de", false)), + isLive = true, + ), + fetchInventory = { + YouTubeSubtitleInventoryResult.Ready( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), + ) + }, + ) + + val result = resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) - assertNull(currentToken()) + val ready = result as YouTubeSubtitleResolution.Ready + assertTrue(ready.track.isLive) + assertEquals("en", ready.track.content.toHttpUrl().queryParameter("lang")) } @Test - fun `restores the outer token after a nested scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("outer", "outer-token")) { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("inner", "inner-token")) { - assertEquals("inner", currentToken()?.visitorData) - } - assertEquals("outer", currentToken()?.visitorData) - } + fun `stream inventory is used when fresh inventory is unavailable`() = runBlocking { + val resolver = StreamYouTubeSubtitleResolver( + streamService = streamService( + listOf(track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false)), + ), + fetchInventory = { YouTubeSubtitleInventoryResult.Unavailable }, + ) - assertNull(currentToken()) + assertTrue(resolver.resolve(selection(YouTubeSubtitleVariant.Manual)) is YouTubeSubtitleResolution.Ready) } @Test - fun `clears the token when the scoped call fails`() { - runCatching { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - error("failed") - } - } + fun `manual and automatic tracks remain distinct`() { + val manual = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en", false) + val automatic = track("https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr", true) - assertNull(currentToken()) + assertTrue(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) + assertFalse(manual.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) + assertTrue(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Auto))) + assertFalse(automatic.matchesYouTubeSubtitle(selection(YouTubeSubtitleVariant.Manual))) } @Test - fun `uses the authenticated provider outside a SABR scope`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) + fun `translated selection preserves source language and requested translation`() { + val track = track( + "https://www.youtube.com/api/timedtext?v=abcdefghijk&lang=en&kind=asr&tlang=de", + true, + ) + val selection = selection(YouTubeSubtitleVariant.Auto).copy( + language = "fr", + sourceLanguage = "en", + translationLanguage = "fr", + ) - assertEquals("auth", currentToken()?.visitorData) - assertEquals("auth-token", currentToken()?.poToken) + assertTrue(track.matchesYouTubeSubtitle(selection)) + val resolved = requireNotNull(track.contentForYouTubeSubtitle(selection)).toHttpUrl() + assertEquals("en", resolved.queryParameter("lang")) + assertEquals("fr", resolved.queryParameter("tlang")) + assertEquals("vtt", resolved.queryParameter("fmt")) } - @Test - fun `prefers the SABR token over the authenticated provider`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) - - TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { - assertEquals("sabr", currentToken()?.visitorData) - assertEquals("sabr-token", currentToken()?.poToken) - } + private fun streamService(subtitles: List, isLive: Boolean = false) = object : StreamService { + override suspend fun getStreamInfo(url: String) = ExtractionResult.Success( + testStreamResponse().copy(subtitles = subtitles, isLive = isLive, isLiveContent = isLive), + ) } - private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( - "MWEB", - "2.20260801.00.00", - "test-user-agent", - Localization("en", "US"), - ContentCountry("US"), - false, + private fun track(url: String, automatic: Boolean) = SubtitleItem( + url = url, + mimeType = "application/ttml+xml", + languageTag = "en", + displayLanguageName = "English", + isAutoGenerated = automatic, ) - private fun token(visitorData: String, playerToken: String) = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = playerToken, - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video-token", - videoBoundPoTokenBytes = byteArrayOf(2), + private fun selection(variant: YouTubeSubtitleVariant) = YouTubeSubtitleSelection( + videoId = "abcdefghijk", + language = "en", + variant = variant, + format = YouTubeSubtitleFormat.Vtt, ) - - private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { - override fun getSessionPoToken( - clientName: String, - clientVersion: String, - userAgent: String?, - localization: Localization, - contentCountry: ContentCountry, - loggedIn: Boolean, - ) = YoutubeSessionPoToken(visitorData, poToken) - } } From 61e86cbf58aed30ca6a499ac0903ae7402cacb70 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 21:48:46 +0200 Subject: [PATCH 039/137] fix: relocate stray playback test files into source sets --- TypetypeYoutubeSessionPoTokenProviderTest.kt | 96 ------------------- .../TypetypeTokenSabrTokenClientTest.kt | 0 ...bePlayerClientFallbackStreamServiceTest.kt | 0 .../YoutubePlayerClientStreamServiceTest.kt | 0 4 files changed, 96 deletions(-) delete mode 100644 TypetypeYoutubeSessionPoTokenProviderTest.kt rename TypetypeTokenSabrTokenClientTest.kt => src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt (100%) rename YoutubePlayerClientFallbackStreamServiceTest.kt => src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt (100%) rename YoutubePlayerClientStreamServiceTest.kt => src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt (100%) diff --git a/TypetypeYoutubeSessionPoTokenProviderTest.kt b/TypetypeYoutubeSessionPoTokenProviderTest.kt deleted file mode 100644 index 3a6a12e4..00000000 --- a/TypetypeYoutubeSessionPoTokenProviderTest.kt +++ /dev/null @@ -1,96 +0,0 @@ -package dev.typetype.server.services - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNull -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Test -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoToken -import org.schabi.newpipe.extractor.services.youtube.YoutubeSessionPoTokenProvider - -class TypetypeYoutubeSessionPoTokenProviderTest { - @AfterEach - fun clearAuthenticatedProvider(): Unit = - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(null) - - @Test - fun `exposes the session token only inside its scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - assertEquals("visitor", currentToken()?.visitorData) - assertEquals("player-token", currentToken()?.poToken) - } - - assertNull(currentToken()) - } - - @Test - fun `restores the outer token after a nested scope`() { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("outer", "outer-token")) { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("inner", "inner-token")) { - assertEquals("inner", currentToken()?.visitorData) - } - assertEquals("outer", currentToken()?.visitorData) - } - - assertNull(currentToken()) - } - - @Test - fun `clears the token when the scoped call fails`() { - runCatching { - TypetypeYoutubeSessionPoTokenProvider.withToken(token("visitor", "player-token")) { - error("failed") - } - } - - assertNull(currentToken()) - } - - @Test - fun `uses the authenticated provider outside a SABR scope`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) - - assertEquals("auth", currentToken()?.visitorData) - assertEquals("auth-token", currentToken()?.poToken) - } - - @Test - fun `prefers the SABR token over the authenticated provider`() { - TypetypeYoutubeSessionPoTokenProvider.configureAuthenticatedProvider(provider("auth", "auth-token")) - - TypetypeYoutubeSessionPoTokenProvider.withToken(token("sabr", "sabr-token")) { - assertEquals("sabr", currentToken()?.visitorData) - assertEquals("sabr-token", currentToken()?.poToken) - } - } - - private fun currentToken() = TypetypeYoutubeSessionPoTokenProvider.getSessionPoToken( - "MWEB", - "2.20260801.00.00", - "test-user-agent", - Localization("en", "US"), - ContentCountry("US"), - false, - ) - - private fun token(visitorData: String, playerToken: String) = SabrTokenBundle( - videoId = "video", - visitorBoundPoToken = playerToken, - visitorBoundPoTokenBytes = byteArrayOf(1), - visitorData = visitorData, - videoBoundPoToken = "video-token", - videoBoundPoTokenBytes = byteArrayOf(2), - ) - - private fun provider(visitorData: String, poToken: String) = object : YoutubeSessionPoTokenProvider { - override fun getSessionPoToken( - clientName: String, - clientVersion: String, - userAgent: String?, - localization: Localization, - contentCountry: ContentCountry, - loggedIn: Boolean, - ) = YoutubeSessionPoToken(visitorData, poToken) - } -} diff --git a/TypetypeTokenSabrTokenClientTest.kt b/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from TypetypeTokenSabrTokenClientTest.kt rename to src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt diff --git a/YoutubePlayerClientFallbackStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt similarity index 100% rename from YoutubePlayerClientFallbackStreamServiceTest.kt rename to src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt diff --git a/YoutubePlayerClientStreamServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt similarity index 100% rename from YoutubePlayerClientStreamServiceTest.kt rename to src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt From afff76c3dfbcf1424ac1f883d78dea77db9c7f2c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 22:01:57 +0200 Subject: [PATCH 040/137] refactor: extract server-services into Gradle module --- build.gradle.kts | 44 +---------- server-core/build.gradle.kts | 1 + .../dev/typetype/server/RequestContext.kt | 13 +++ .../server/models/AdminSettingsItem.kt | 0 .../typetype/server/models/ErrorResponse.kt | 0 .../typetype/server/models/ProxyResponse.kt | 0 server-services/build.gradle.kts | 79 +++++++++++++++++++ .../server/models/HomeRecommendationPool.kt | 0 .../server/services/AccessControlProfile.kt | 0 .../server/services/AccessControlService.kt | 0 .../AccessControlledResponseFilters.kt | 12 +-- .../services/AccessControlledVideoFilters.kt | 4 +- .../server/services/ActiveSessionService.kt | 0 .../services/AdminManagedAccessService.kt | 0 .../server/services/AdminSettingsService.kt | 0 .../server/services/AdminUserLookupService.kt | 0 .../server/services/AllowedChannelsService.kt | 0 .../services/AllowedPlaylistsService.kt | 0 .../services/AudioOnlyHlsFallbackStream.kt | 2 +- .../server/services/AudioOnlyMediaToken.kt | 0 .../services/AudioOnlyMediaTokenService.kt | 0 .../server/services/AudioOnlyStreamKind.kt | 0 .../services/AudioOnlyStreamResolver.kt | 0 .../services/AudioOnlyStreamSelection.kt | 0 .../services/AudioOnlyStreamSelector.kt | 0 .../server/services/AvatarBinaryCache.kt | 0 .../server/services/BilibiliRangeProxy.kt | 2 +- .../server/services/BilibiliRelatedService.kt | 2 +- .../services/BilibiliTrendingService.kt | 0 .../server/services/BlockedContentFilters.kt | 6 +- .../server/services/BlockedContentProfile.kt | 2 +- .../server/services/BlockedService.kt | 2 +- .../server/services/BugReportRowMapper.kt | 2 +- .../server/services/BugReportService.kt | 0 .../server/services/BugReportStatusFlow.kt | 2 +- .../services/BugReportStatusUpdateResult.kt | 0 .../server/services/BugReportValidation.kt | 2 +- .../server/services/BulletCommentService.kt | 0 .../server/services/CachedChannelService.kt | 0 .../server/services/CachedCommentService.kt | 0 .../server/services/CachedManifestService.kt | 0 .../services/CachedNativeManifestService.kt | 0 .../server/services/CachedPodcastService.kt | 0 .../services/CachedPublicPlaylistService.kt | 0 .../server/services/CachedSearchService.kt | 0 .../server/services/CachedStreamService.kt | 0 .../services/CachedSuggestionService.kt | 0 .../server/services/CachedTrendingService.kt | 0 .../ChannelNotificationPreferenceService.kt | 4 +- .../server/services/ChannelResponseMappers.kt | 12 +-- .../services/ChannelResultItemMappers.kt | 2 +- .../server/services/ChannelService.kt | 0 .../server/services/ChannelTabExtraction.kt | 4 +- .../server/services/ChannelTabResolver.kt | 4 +- .../server/services/CommentService.kt | 0 .../server/services/CustomAvatarService.kt | 0 .../services/CustomAvatarUploadReader.kt | 0 .../DashManifestStreamCompatibility.kt | 2 +- .../typetype/server/services/DeArrowClient.kt | 0 .../server/services/DeArrowService.kt | 0 .../services/DownloaderGatewayService.kt | 0 .../services/ExtractionErrorSanitizer.kt | 0 .../server/services/ExtractionResilience.kt | 2 +- .../server/services/FavoriteItemFallback.kt | 2 +- .../server/services/FavoritesService.kt | 0 .../server/services/GitHubIssueService.kt | 0 .../server/services/HistoryProgressMapper.kt | 2 +- .../server/services/HistoryService.kt | 0 .../server/services/HlsManifestCache.kt | 2 +- .../server/services/HlsManifestRewrite.kt | 6 +- .../server/services/HlsManifestService.kt | 0 .../services/HomeRecommendationApiArgs.kt | 0 .../services/HomeRecommendationBuilder.kt | 0 .../HomeRecommendationCandidateLimits.kt | 0 .../HomeRecommendationCandidatePool.kt | 0 .../HomeRecommendationCandidateService.kt | 0 .../services/HomeRecommendationContext.kt | 0 .../HomeRecommendationContextualBandit.kt | 0 .../services/HomeRecommendationCursor.kt | 0 .../services/HomeRecommendationCursorCodec.kt | 0 .../HomeRecommendationCursorFactory.kt | 0 .../HomeRecommendationCursorMemory.kt | 0 .../HomeRecommendationCursorPageIndex.kt | 0 .../HomeRecommendationCursorPayload.kt | 0 .../services/HomeRecommendationCursorState.kt | 0 .../services/HomeRecommendationDecay.kt | 0 .../services/HomeRecommendationDecision.kt | 0 .../HomeRecommendationDelayedCreditBuilder.kt | 0 .../services/HomeRecommendationDeviceClass.kt | 0 .../HomeRecommendationDeviceScorer.kt | 0 .../HomeRecommendationDiscoveryAssembler.kt | 0 .../HomeRecommendationEngagementSignals.kt | 0 .../HomeRecommendationEngagementSplit.kt | 0 ...RecommendationEngagementSplitCalculator.kt | 0 .../HomeRecommendationEventAnalyzer.kt | 0 .../HomeRecommendationExplorationBandit.kt | 0 ...eRecommendationExplorationQueryProvider.kt | 0 .../HomeRecommendationExploreBonus.kt | 0 .../HomeRecommendationFeedHistoryPenalty.kt | 0 .../services/HomeRecommendationJitter.kt | 0 .../HomeRecommendationLanguageGate.kt | 0 .../HomeRecommendationLiveTitleDetector.kt | 0 .../services/HomeRecommendationMetrics.kt | 0 .../services/HomeRecommendationMixEngine.kt | 0 .../services/HomeRecommendationMixRound.kt | 0 .../services/HomeRecommendationMixState.kt | 0 .../services/HomeRecommendationMixer.kt | 0 .../HomeRecommendationMomentumMemory.kt | 0 .../services/HomeRecommendationNextCursor.kt | 0 .../HomeRecommendationNoopStreamService.kt | 0 .../HomeRecommendationOfflineEvaluator.kt | 0 .../server/services/HomeRecommendationPage.kt | 0 .../services/HomeRecommendationPageBuilder.kt | 0 .../services/HomeRecommendationPageDedup.kt | 0 .../HomeRecommendationPersonaDrift.kt | 0 .../HomeRecommendationPersonaState.kt | 0 .../HomeRecommendationPersonaTracker.kt | 0 .../HomeRecommendationPersonalization.kt | 0 .../services/HomeRecommendationPickState.kt | 0 .../services/HomeRecommendationPicker.kt | 0 .../services/HomeRecommendationPoolBuilder.kt | 0 .../services/HomeRecommendationPoolCache.kt | 0 .../services/HomeRecommendationPoolMode.kt | 0 .../HomeRecommendationPoolResolver.kt | 0 ...eRecommendationPoolResolverDependencies.kt | 0 .../HomeRecommendationPoolResolverState.kt | 0 .../HomeRecommendationPoolRotation.kt | 0 .../services/HomeRecommendationPoolScorer.kt | 0 .../services/HomeRecommendationPoolWeights.kt | 0 .../services/HomeRecommendationProfile.kt | 0 .../HomeRecommendationQuotaPlanner.kt | 0 ...meRecommendationRelatedCandidateService.kt | 0 .../services/HomeRecommendationScoredVideo.kt | 0 .../services/HomeRecommendationScoring.kt | 0 ...omeRecommendationSearchCandidateFetcher.kt | 0 .../services/HomeRecommendationSelection.kt | 0 .../services/HomeRecommendationSelector.kt | 0 .../services/HomeRecommendationSemanticKey.kt | 0 .../services/HomeRecommendationService.kt | 0 .../HomeRecommendationSessionContext.kt | 0 .../HomeRecommendationSessionIntentScorer.kt | 0 .../HomeRecommendationShortProfileFallback.kt | 0 ...meRecommendationShortSubscriptionSource.kt | 0 .../services/HomeRecommendationShortsAudit.kt | 0 ...omeRecommendationShortsCandidateService.kt | 0 .../HomeRecommendationShortsClassifier.kt | 0 .../HomeRecommendationShortsDebugInfo.kt | 0 .../HomeRecommendationShortsDeduplicator.kt | 0 .../HomeRecommendationShortsDefaults.kt | 0 .../HomeRecommendationShortsFallback.kt | 0 .../HomeRecommendationShortsHistorySignals.kt | 0 .../HomeRecommendationShortsProfileFit.kt | 0 .../HomeRecommendationShortsQueryFactory.kt | 0 .../HomeRecommendationShortsRefresher.kt | 0 .../HomeRecommendationShortsSeenMemory.kt | 0 .../HomeRecommendationShortsSignals.kt | 0 .../HomeRecommendationShortsSourceWeights.kt | 0 .../HomeRecommendationShortsSources.kt | 0 .../HomeRecommendationSignalContext.kt | 0 .../HomeRecommendationSignalProfileBuilder.kt | 0 .../HomeRecommendationSourceBandit.kt | 0 .../services/HomeRecommendationSourceTag.kt | 0 .../HomeRecommendationStateMachine.kt | 0 .../services/HomeRecommendationTaggedVideo.kt | 0 .../services/HomeRecommendationTargetPlan.kt | 0 .../HomeRecommendationTemporalBoost.kt | 0 .../HomeRecommendationThemeExtractor.kt | 0 .../HomeRecommendationTokenNormalizer.kt | 0 .../services/HomeRecommendationTopicPairs.kt | 0 .../HomeRecommendationUserSignalService.kt | 0 .../services/HomeRecommendationWarmup.kt | 0 .../HomeRecommendationWarmupService.kt | 0 .../server/services/HomeWarmupTracker.kt | 2 +- .../server/services/InstanceService.kt | 0 .../server/services/InternalHealthService.kt | 0 .../server/services/LiveStreamTypes.kt | 12 +-- .../services/ManifestRepresentationIds.kt | 8 +- .../server/services/ManifestService.kt | 0 .../server/services/NativeManifestBuilder.kt | 2 +- .../server/services/NativeManifestService.kt | 0 .../server/services/NativeManifestUtils.kt | 10 +-- .../services/NicoNicoTrendingService.kt | 0 .../server/services/NicoVideoProxyService.kt | 4 +- .../server/services/NotificationsService.kt | 0 .../server/services/OkHttpProxyService.kt | 8 +- .../OkHttpYoutubeRemoteBrowserClient.kt | 0 .../server/services/OpenMojiProxyService.kt | 0 .../typetype/server/services/PageSurrogate.kt | 6 +- .../services/PipePipeBackupImporterService.kt | 0 .../server/services/PipePipeBackupLimits.kt | 0 .../PipePipeBackupPersisterService.kt | 0 .../services/PipePipeBackupReadModels.kt | 0 .../services/PipePipeBackupRestoreResult.kt | 0 .../services/PipePipeBackupSnapshotItem.kt | 0 .../PipePipeBackupSnapshotTimeMode.kt | 0 .../services/PipePipeBackupSqliteMeta.kt | 0 .../services/PipePipeBackupSqliteReader.kt | 0 .../server/services/PipePipeBackupTimeMode.kt | 0 .../PipePipeBackupTimelineNormalizer.kt | 0 .../services/PipePipeBackupUploadWriter.kt | 0 .../services/PipePipeBackupValidators.kt | 0 .../services/PipePipeBackupZipExtractor.kt | 0 .../services/PipePipeBulletCommentService.kt | 0 .../server/services/PipePipeChannelService.kt | 0 .../server/services/PipePipeCommentService.kt | 2 +- .../PipePipePodcastEpisodesService.kt | 2 +- .../server/services/PipePipePodcastService.kt | 0 .../services/PipePipePublicPlaylistService.kt | 0 .../server/services/PipePipeSearchService.kt | 0 .../server/services/PipePipeStreamService.kt | 4 +- .../services/PipePipeSuggestionService.kt | 0 .../services/PipePipeTrendingService.kt | 0 .../services/PlaylistResultItemMappers.kt | 2 +- .../server/services/PlaylistService.kt | 0 .../services/PlaylistVideoProgressMapper.kt | 6 +- .../server/services/PodcastItemMappers.kt | 6 +- .../server/services/PodcastService.kt | 0 .../server/services/PresenceKeyService.kt | 0 .../server/services/PresenceService.kt | 0 .../server/services/PresenceTokenCodec.kt | 4 +- .../server/services/ProgressService.kt | 0 .../services/ProviderHlsManifestRewrite.kt | 2 +- .../services/ProviderMediaHandleService.kt | 12 +-- .../server/services/ProxyHttpExecutor.kt | 4 +- .../typetype/server/services/ProxyService.kt | 2 +- .../server/services/PublicCachePolicy.kt | 2 +- .../server/services/PublicExtractionCache.kt | 2 +- .../server/services/PublicHlsManifestToken.kt | 0 .../services/PublicHlsManifestTokenService.kt | 0 .../services/PublicPlaylistItemMappers.kt | 4 +- .../server/services/PublicPlaylistService.kt | 0 .../server/services/PublishedAtMapper.kt | 2 +- .../server/services/PushDeviceRegistry.kt | 6 +- .../services/PushNotificationDeliveryStore.kt | 4 +- .../services/PushNotificationScheduler.kt | 2 +- .../services/PushNotificationService.kt | 8 +- .../services/PushNotificationSupport.kt | 2 +- .../services/RecommendationEventService.kt | 0 .../RecommendationFeedHistoryEntry.kt | 0 .../RecommendationFeedHistoryService.kt | 0 .../services/RecommendationFeedbackService.kt | 0 .../RecommendationFeedbackSignalService.kt | 0 .../RecommendationInterestBatchUpdater.kt | 0 .../RecommendationInterestProfileService.kt | 0 .../services/RecommendationInterestService.kt | 0 .../services/RecommendationInterestWeight.kt | 0 .../RecommendationOnboardingCatalog.kt | 0 .../RecommendationOnboardingCreativeTopics.kt | 0 ...RecommendationOnboardingEducationTopics.kt | 0 ...mmendationOnboardingEntertainmentTopics.kt | 0 .../RecommendationOnboardingGamingTopics.kt | 0 ...mmendationOnboardingHealthFitnessTopics.kt | 0 ...RecommendationOnboardingLifestyleTopics.kt | 0 .../RecommendationOnboardingMusicTopics.kt | 0 .../RecommendationOnboardingNewsTopics.kt | 0 ...ecommendationOnboardingPreferencesStore.kt | 0 ...mmendationOnboardingScienceNatureTopics.kt | 0 .../RecommendationOnboardingSeedService.kt | 0 .../RecommendationOnboardingService.kt | 0 .../RecommendationOnboardingStateStore.kt | 0 ...ecommendationOnboardingTechnologyTopics.kt | 0 .../RecommendationOnboardingTopicGroups.kt | 0 .../services/RecommendationPrivacyService.kt | 0 .../services/RecommendationServiceId.kt | 2 +- .../services/RecommendationTopicTokenizer.kt | 0 .../server/services/RelatedItemMappers.kt | 12 +-- .../services/RetryingProxyInputStream.kt | 2 +- .../server/services/RssDocumentRenderer.kt | 2 +- .../server/services/RssFeedAdminRepository.kt | 2 +- .../services/RssFeedManagementService.kt | 2 +- .../server/services/RssFeedReaderService.kt | 2 +- .../server/services/RssFeedRepository.kt | 4 +- .../server/services/RssFeedRowMapper.kt | 4 +- .../typetype/server/services/RssFeedSecret.kt | 2 +- .../server/services/RssFeedSelections.kt | 4 +- .../server/services/RssFeedThrottle.kt | 2 +- .../server/services/RssVideoMetadata.kt | 2 +- .../server/services/RssVideoTypeFilter.kt | 2 +- .../server/services/SavedPlaylistService.kt | 0 .../server/services/SavedPlaylistWrite.kt | 2 +- .../server/services/SearchContentKind.kt | 8 +- .../server/services/SearchFilterMappers.kt | 16 ++-- .../server/services/SearchHistoryService.kt | 0 .../server/services/SearchPageMappers.kt | 4 +- .../typetype/server/services/SearchService.kt | 0 .../typetype/server/services/ServiceIds.kt | 0 .../server/services/SettingsService.kt | 4 +- .../services/SignedHlsManifestCookie.kt | 0 .../server/services/SignedHlsManifestToken.kt | 0 .../services/SignedHlsManifestTokenService.kt | 0 .../services/StoryboardProxyDetector.kt | 0 .../services/StreamAudioContractResolver.kt | 0 .../server/services/StreamCacheTtlResolver.kt | 2 +- .../services/StreamExtractionErrorMapper.kt | 2 +- .../server/services/StreamInfoMappers.kt | 4 +- .../server/services/StreamMediaItemMappers.kt | 8 +- .../server/services/StreamTimelineMappers.kt | 4 +- .../services/StreamYouTubeSubtitleResolver.kt | 8 +- .../services/SubscriptionAvatarRepairer.kt | 0 .../services/SubscriptionFeedAvailability.kt | 2 +- .../services/SubscriptionFeedBuilder.kt | 4 +- .../SubscriptionFeedCacheInvalidator.kt | 0 .../services/SubscriptionFeedCacheKeys.kt | 0 .../services/SubscriptionFeedOrderer.kt | 6 +- .../SubscriptionFeedSelectionStore.kt | 4 +- .../services/SubscriptionFeedService.kt | 10 +-- .../services/SubscriptionFeedSnapshot.kt | 12 +-- .../services/SubscriptionFeedSnapshotStore.kt | 2 +- .../services/SubscriptionGroupResults.kt | 0 .../services/SubscriptionGroupsService.kt | 0 .../server/services/SubscriptionSelection.kt | 0 .../SubscriptionShortsBlendService.kt | 0 .../services/SubscriptionShortsFeedService.kt | 0 .../SubscriptionShortsSignalService.kt | 0 .../server/services/SubscriptionsService.kt | 0 .../server/services/SuggestionService.kt | 0 .../server/services/TrendingService.kt | 0 .../server/services/TypeTypeBackupService.kt | 0 .../services/UnifiedPushEndpointValidator.kt | 4 +- .../server/services/UnifiedPushSender.kt | 6 +- .../typetype/server/services/UrlValidator.kt | 14 ++-- .../server/services/UserAdminService.kt | 0 .../UserVideoMetadataRepairService.kt | 0 .../server/services/VideoItemSchedule.kt | 11 +++ .../server/services/VideoMetadataItem.kt | 0 .../server/services/VideoMetadataResolver.kt | 0 .../server/services/WatchLaterService.kt | 0 .../server/services/YouTubeChannelTabSort.kt | 2 +- .../server/services/YouTubeSubtitleCache.kt | 2 +- .../services/YouTubeSubtitleContentFetcher.kt | 10 +-- .../services/YouTubeSubtitleContract.kt | 22 +++--- .../YouTubeSubtitleDeliveryService.kt | 2 +- .../server/services/YouTubeSubtitleService.kt | 4 +- .../server/services/YoutubePlayerClient.kt | 2 +- ...outubePlayerClientFallbackStreamService.kt | 2 +- .../services/YoutubePlayerClientScope.kt | 2 +- .../YoutubePlayerClientStreamService.kt | 2 +- .../services/YoutubeRemoteBrowserAudit.kt | 0 .../YoutubeRemoteBrowserBridgeListener.kt | 0 .../services/YoutubeRemoteBrowserClient.kt | 0 .../YoutubeRemoteBrowserCompleteResult.kt | 0 .../services/YoutubeRemoteBrowserConfig.kt | 0 .../services/YoutubeRemoteBrowserService.kt | 0 .../YoutubeRemoteBrowserSessionStore.kt | 0 .../YoutubeRemoteLoginReadinessService.kt | 0 .../services/YoutubeScopedChannelService.kt | 0 .../services/YoutubeScopedCommentService.kt | 0 .../YoutubeScopedPublicPlaylistService.kt | 0 .../services/YoutubeScopedSearchService.kt | 0 .../services/YoutubeScopedStreamService.kt | 0 .../YoutubeScopedSuggestionService.kt | 0 .../services/YoutubeScopedTrendingService.kt | 0 .../YoutubeSessionHlsManifestService.kt | 0 .../YoutubeSessionSabrStreamService.kt | 8 +- .../services/YoutubeSessionStreamService.kt | 2 +- .../server/services/YoutubeTakeoutFactory.kt | 0 .../services/YoutubeTakeoutImportCache.kt | 0 .../YoutubeTakeoutImportCleanupService.kt | 0 .../services/YoutubeTakeoutImportJobEngine.kt | 0 .../YoutubeTakeoutImportJobService.kt | 0 .../services/YoutubeTakeoutImporterService.kt | 0 .../services/YoutubeTakeoutParserService.kt | 0 .../YoutubeTakeoutPreviewLookupService.kt | 0 .../services/YoutubeTakeoutPreviewService.kt | 0 .../YoutubeTakeoutSignalImportService.kt | 0 settings.gradle.kts | 1 + .../typetype/server/RequestObservability.kt | 7 +- .../server/services/VideoItemSchedule.kt | 11 --- 368 files changed, 327 insertions(+), 280 deletions(-) create mode 100644 server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ErrorResponse.kt (100%) rename {src => server-core/src}/main/kotlin/dev/typetype/server/models/ProxyResponse.kt (100%) create mode 100644 server-services/build.gradle.kts rename {src => server-services/src}/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AccessControlService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt (60%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt (58%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt (86%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt (69%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BlockedService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BugReportService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt (91%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BugReportValidation.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/BulletCommentService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedChannelService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedCommentService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedSearchService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedStreamService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt (78%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt (80%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt (86%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt (79%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CommentService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt (86%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/DeArrowClient.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/DeArrowService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt (72%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/FavoritesService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HistoryService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt (88%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt (89%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HlsManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt (95%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/InstanceService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/InternalHealthService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt (61%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt (66%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt (99%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NativeManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt (73%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/NotificationsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PageSurrogate.kt (90%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt (80%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt (88%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt (79%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PodcastService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PresenceService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt (90%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ProgressService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt (94%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ProxyService.kt (89%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt (74%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt (76%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PushNotificationService.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt (88%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt (87%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RetryingProxyInputStream.kt (99%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt (99%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt (89%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt (91%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt (88%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt (92%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt (95%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt (91%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt (87%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SearchContentKind.kt (76%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt (82%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt (86%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SearchService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/ServiceIds.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SettingsService.kt (93%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt (94%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt (95%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt (92%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt (80%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt (92%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt (86%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt (92%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt (93%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/SuggestionService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/TrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt (96%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt (95%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/UrlValidator.kt (90%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/UserAdminService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt (100%) create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/WatchLaterService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt (87%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt (93%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt (85%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt (98%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt (93%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt (66%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt (93%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt (95%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt (91%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt (87%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt (97%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt (100%) delete mode 100644 src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt diff --git a/build.gradle.kts b/build.gradle.kts index 3bd98cb5..22e1d84a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { implementation(project(":server-domain")) implementation(project(":server-portability")) implementation(project(":server-playback")) + implementation(project(":server-services")) implementation(project(":server-sabr")) implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) @@ -74,46 +75,6 @@ dependencies { testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") testImplementation("org.testcontainers:testcontainers-junit-jupiter:2.0.5") } -val buildInfoVersion = version.toString().trim().takeUnless { it.isBlank() || it == "unspecified" } ?: "0.0.0-dev" -fun gitRevisionOrUnknown(): String = runCatching { - providers.exec { commandLine("git", "rev-parse", "HEAD") } - .standardOutput - .asText - .get() - .trim() - .ifBlank { "unknown" } -}.getOrElse { "unknown" } - -val buildInfoRevision = providers.environmentVariable("GITHUB_SHA") - .map { it.trim().ifBlank { "unknown" } } - .getOrElse(gitRevisionOrUnknown()) -val buildInfoShortRevision = buildInfoRevision.takeIf { it != "unknown" }?.take(12) ?: "unknown" -val buildInfoBuildTime = providers.environmentVariable("BUILD_TIME") - .orElse(providers.provider { Instant.now().toString() }) - .get() -val generatedBuildInfoDir = layout.buildDirectory.dir("generated/sources/buildInfo/main") -val generateBuildInfo = tasks.register("generateBuildInfo") { - inputs.property("version", buildInfoVersion) - inputs.property("revision", buildInfoRevision) - inputs.property("shortRevision", buildInfoShortRevision) - inputs.property("buildTime", buildInfoBuildTime) - outputs.dir(generatedBuildInfoDir) - doLast { - val output = generatedBuildInfoDir.get().file("dev/typetype/server/BuildInfo.kt").asFile - output.parentFile.mkdirs() - output.writeText(""" - package dev.typetype.server - - object BuildInfo { - const val VERSION: String = "${buildInfoVersion.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val REVISION: String = "${buildInfoRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val SHORT_REVISION: String = "${buildInfoShortRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" - const val BUILD_TIME: String = "${buildInfoBuildTime.replace("\\", "\\\\").replace("\"", "\\\"")}" - } - """.trimIndent()) - } -} - tasks.test { useJUnitPlatform { // network-tagged tests are off by default. Flip on for the live SABR probe: @@ -180,11 +141,8 @@ tasks.check { kotlin { jvmToolchain(25) - sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } } -tasks.named("compileKotlin") { dependsOn(generateBuildInfo) } - tasks.withType().configureEach { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } diff --git a/server-core/build.gradle.kts b/server-core/build.gradle.kts index c0035dab..0514878c 100644 --- a/server-core/build.gradle.kts +++ b/server-core/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") } kotlin { diff --git a/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt new file mode 100644 index 00000000..ecbf335d --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt @@ -0,0 +1,13 @@ +package dev.typetype.server + +import kotlinx.coroutines.ThreadContextElement +import kotlinx.coroutines.asContextElement + +private val requestIdContext = ThreadLocal() + +const val REQUEST_ID_HEADER = "X-Request-ID" + +fun currentRequestId(): String? = requestIdContext.get() + +fun requestContextElement(requestId: String?): ThreadContextElement = + requestIdContext.asContextElement(requestId) diff --git a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ErrorResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt rename to server-core/src/main/kotlin/dev/typetype/server/models/ProxyResponse.kt diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts new file mode 100644 index 00000000..fedf7105 --- /dev/null +++ b/server-services/build.gradle.kts @@ -0,0 +1,79 @@ +import java.time.Instant +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-domain")) + api(project(":server-playback")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-downloader")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-client-core-jvm:3.5.2") + implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.squareup.okhttp3:okhttp:5.5.0") + implementation("org.json:json:20260814") + implementation("org.slf4j:slf4j-api:2.0.16") + implementation("org.jetbrains.exposed:exposed-core:1.5.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") +} + +kotlin { + jvmToolchain(25) +} + +val applicationVersion = providers.gradleProperty("appVersion").get() +val buildInfoVersion = applicationVersion.trim().takeUnless { it.isBlank() || it == "unspecified" } ?: "0.0.0-dev" +fun gitRevisionOrUnknown(): String = runCatching { + providers.exec { commandLine("git", "rev-parse", "HEAD") } + .standardOutput + .asText + .get() + .trim() + .ifBlank { "unknown" } +}.getOrElse { "unknown" } +val buildInfoRevision = providers.environmentVariable("GITHUB_SHA") + .map { it.trim().ifBlank { "unknown" } } + .getOrElse(gitRevisionOrUnknown()) +val buildInfoShortRevision = buildInfoRevision.takeIf { it != "unknown" }?.take(12) ?: "unknown" +val buildInfoBuildTime = providers.environmentVariable("BUILD_TIME") + .orElse(providers.provider { Instant.now().toString() }) + .get() +val generatedBuildInfoDir = layout.buildDirectory.dir("generated/sources/buildInfo/main") +val generateBuildInfo = tasks.register("generateBuildInfo") { + inputs.property("version", buildInfoVersion) + inputs.property("revision", buildInfoRevision) + inputs.property("shortRevision", buildInfoShortRevision) + inputs.property("buildTime", buildInfoBuildTime) + outputs.dir(generatedBuildInfoDir) + doLast { + val output = generatedBuildInfoDir.get().file("dev/typetype/server/BuildInfo.kt").asFile + output.parentFile.mkdirs() + output.writeText(""" + package dev.typetype.server + + object BuildInfo { + const val VERSION: String = "${buildInfoVersion.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val REVISION: String = "${buildInfoRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val SHORT_REVISION: String = "${buildInfoShortRevision.replace("\\", "\\\\").replace("\"", "\\\"")}" + const val BUILD_TIME: String = "${buildInfoBuildTime.replace("\\", "\\\\").replace("\"", "\\\"")}" + } + """.trimIndent()) + } +} + +sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } + +tasks.named("compileKotlin") { dependsOn(generateBuildInfo) } diff --git a/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt b/server-services/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt rename to server-services/src/main/kotlin/dev/typetype/server/models/HomeRecommendationPool.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlProfile.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AccessControlService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt similarity index 60% rename from src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt index c5b81acd..bac06e5b 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledResponseFilters.kt @@ -7,28 +7,28 @@ import dev.typetype.server.models.PublicPlaylistResponse import dev.typetype.server.models.SearchPageResponse import dev.typetype.server.models.StreamResponse -internal fun SearchPageResponse.filterAllowed(profile: AccessControlProfile): SearchPageResponse = copy( +fun SearchPageResponse.filterAllowed(profile: AccessControlProfile): SearchPageResponse = copy( items = items.filterAllowed(profile), channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, playlists = playlists.filter { profile.allowsPlaylist(it.url) || profile.allowsChannel(url = "", name = it.uploaderName) }, ) -internal fun StreamResponse.filterAllowed(profile: AccessControlProfile): StreamResponse = copy( +fun StreamResponse.filterAllowed(profile: AccessControlProfile): StreamResponse = copy( relatedStreams = relatedStreams.filterAllowed(profile), ) -internal fun PublicPlaylistResponse.filterAllowed(profile: AccessControlProfile): PublicPlaylistResponse = copy( +fun PublicPlaylistResponse.filterAllowed(profile: AccessControlProfile): PublicPlaylistResponse = copy( videos = if (profile.allowsPlaylist(playlist.url)) videos else videos.filterAllowed(profile), ) -internal fun ChannelResponse.filterAllowed(profile: AccessControlProfile): ChannelResponse = copy( +fun ChannelResponse.filterAllowed(profile: AccessControlProfile): ChannelResponse = copy( videos = videos.filterAllowed(profile), ) -internal fun ChannelPlaylistsResponse.filterAllowed(profile: AccessControlProfile): ChannelPlaylistsResponse = copy( +fun ChannelPlaylistsResponse.filterAllowed(profile: AccessControlProfile): ChannelPlaylistsResponse = copy( playlists = playlists.filter { profile.allowsPlaylist(it.url) || profile.allowsChannel(url = "", name = it.uploaderName) }, ) -internal fun HomeRecommendationsResponse.filterAllowed(profile: AccessControlProfile): HomeRecommendationsResponse = copy( +fun HomeRecommendationsResponse.filterAllowed(profile: AccessControlProfile): HomeRecommendationsResponse = copy( items = items.filterAllowed(profile), ) diff --git a/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt similarity index 58% rename from src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt index 0a222b4d..79507b52 100644 --- a/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AccessControlledVideoFilters.kt @@ -2,8 +2,8 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem -internal fun List.filterAllowed(profile: AccessControlProfile): List = +fun List.filterAllowed(profile: AccessControlProfile): List = filter { profile.allowsUploader(url = it.uploaderUrl, name = it.uploaderName) } -internal fun VideoItem.isAllowedBy(profile: AccessControlProfile): Boolean = +fun VideoItem.isAllowedBy(profile: AccessControlProfile): Boolean = profile.allowsUploader(url = uploaderUrl, name = uploaderName) diff --git a/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ActiveSessionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminManagedAccessService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AdminUserLookupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AllowedPlaylistsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt index b9d9b005..e24cc981 100644 --- a/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyHlsFallbackStream.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.AudioStreamItem -internal fun hlsFallbackStream(url: String): AudioStreamItem = AudioStreamItem( +fun hlsFallbackStream(url: String): AudioStreamItem = AudioStreamItem( url = url, mimeType = "application/vnd.apple.mpegurl", format = "HLS", diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyMediaTokenService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamKind.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AudioOnlyStreamSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/AvatarBinaryCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 0f04f09a..3b502cf2 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -8,7 +8,7 @@ import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 -internal fun readBilibiliRangeWithRetry( +fun readBilibiliRangeWithRetry( execute: (Request) -> Response, request: Request, checkActive: () -> Unit = {}, diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt index 38a55888..a7e67476 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliRelatedService.kt @@ -16,7 +16,7 @@ private const val RELATED_BASE_URL = "https://api.bilibili.com/x/web-interface/a private const val SPACE_BASE_URL = "https://space.bilibili.com/" private val BVID_REGEX = Regex("""/(BV[0-9A-Za-z]+)""") -internal class BilibiliRelatedService( +class BilibiliRelatedService( private val relatedLookupOverride: (suspend (String) -> Map)? = null, ) { private val uploaderUrlCache = BoundedExpiringCache>( diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BilibiliTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt similarity index 69% rename from src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt index 0048911d..76d52af9 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt @@ -4,19 +4,19 @@ import dev.typetype.server.models.HomeRecommendationsResponse import dev.typetype.server.models.SearchPageResponse import dev.typetype.server.models.StreamResponse -internal fun HomeRecommendationsResponse.filterBlocked(profile: BlockedContentProfile): HomeRecommendationsResponse = copy( +fun HomeRecommendationsResponse.filterBlocked(profile: BlockedContentProfile): HomeRecommendationsResponse = copy( items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, ) -internal fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( +fun SearchPageResponse.filterBlocked(profile: BlockedContentProfile): SearchPageResponse = copy( items = items.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, channels = channels.filter { profile.allowsChannel(url = it.url, name = it.name) }, playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, ) -internal fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse = copy( +fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse = copy( relatedStreams = relatedStreams.filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt index 9595097f..6447fff0 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentProfile.kt @@ -38,7 +38,7 @@ data class BlockedContentProfile( private val YOUTUBE_VIDEO_ID = Regex("(?:[?&]v=|/(?:shorts|embed|live)/|youtu\\.be/)([A-Za-z0-9_-]{6,})", RegexOption.IGNORE_CASE) -internal fun normalizeBlockedVideoKey(value: String): String { +fun normalizeBlockedVideoKey(value: String): String { val trimmed = value.trim() val youtubeId = YOUTUBE_VIDEO_ID.find(trimmed)?.groupValues?.get(1) return youtubeId?.let { "youtube:video:$it" } ?: normalizeChannelKey(trimmed) diff --git a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BlockedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt index d3c72031..31472655 100644 --- a/src/main/kotlin/dev/typetype/server/services/BlockedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedService.kt @@ -161,7 +161,7 @@ class BlockedService { } } -internal fun containsBlockedKeyword(title: String, keyword: String): Boolean { +fun containsBlockedKeyword(title: String, keyword: String): Boolean { val normalizedKeyword = normalizeBlockedKeyword(keyword) return normalizedKeyword.isNotEmpty() && normalizeBlockedKeyword(title).contains(normalizedKeyword) } diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt index d79f937a..8f5c3f50 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportRowMapper.kt @@ -7,7 +7,7 @@ import dev.typetype.server.models.BugReportContextItem import kotlinx.serialization.json.Json import org.jetbrains.exposed.v1.core.ResultRow -internal object BugReportRowMapper { +object BugReportRowMapper { fun toAdminListItem(row: ResultRow, userEmail: String): AdminBugReportItem = AdminBugReportItem( id = row[BugReportsTable.id], category = row[BugReportsTable.category], diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BugReportService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt index cc5a8cdc..62edbf12 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusFlow.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object BugReportStatusFlow { +object BugReportStatusFlow { private val nextStatus: Map = mapOf( "new" to "triaged", "triaged" to "in_progress", diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportStatusUpdateResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt index e1c6d485..4fe95c7a 100644 --- a/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BugReportValidation.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.BugReportContextItem -internal object BugReportValidation { +object BugReportValidation { private val categories = setOf("player", "audio_language", "subtitles", "ui", "functionality") private val statuses = setOf("new", "triaged", "in_progress", "fixed", "closed") diff --git a/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/BulletCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedNativeManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedPodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedPublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CachedTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt index 10968e4e..f435fffc 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt @@ -10,7 +10,7 @@ import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class ChannelNotificationPreferenceService( +class ChannelNotificationPreferenceService( private val subscriptionsService: SubscriptionsService, ) { suspend fun list(userId: String): List { @@ -76,7 +76,7 @@ internal class ChannelNotificationPreferenceService( } } -internal sealed interface PreferenceUpdateResult { +sealed interface PreferenceUpdateResult { data class Updated(val preference: ChannelNotificationPreference) : PreferenceUpdateResult data object NotSubscribed : PreferenceUpdateResult } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt similarity index 78% rename from src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt index 3ef4acf3..d358342a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResponseMappers.kt @@ -9,7 +9,7 @@ import org.schabi.newpipe.extractor.channel.ChannelTabInfo import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( +fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( name = name ?: "", description = description ?: "", avatarUrl = avatarUrl ?: "", @@ -20,7 +20,7 @@ internal fun ChannelInfo.toChannelResponse(): ChannelResponse = ChannelResponse( nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( +fun InfoItemsPage.toChannelResponse(): ChannelResponse = ChannelResponse( name = "", description = "", avatarUrl = "", @@ -31,7 +31,7 @@ internal fun InfoItemsPage.toChannelResponse(): ChannelResponse nextpage = nextPage?.toCursor(), ) -internal fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): ChannelResponse = ChannelResponse( +fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): ChannelResponse = ChannelResponse( name = metadata?.name ?: name ?: "", description = metadata?.description ?: "", avatarUrl = metadata?.avatarUrl ?: "", @@ -42,7 +42,7 @@ internal fun ChannelTabInfo.toChannelTabResponse(metadata: ChannelInfo? = null): nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( +fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = ChannelResponse( name = "", description = "", avatarUrl = "", @@ -53,12 +53,12 @@ internal fun InfoItemsPage.toChannelTabResponse(): ChannelResponse = C nextpage = nextPage?.toCursor(), ) -internal fun ChannelTabInfo.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( +fun ChannelTabInfo.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( playlists = relatedItems.filterIsInstance().map { it.toPlaylistResultItem() }, nextpage = nextPage?.toCursor(), ) -internal fun InfoItemsPage.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( +fun InfoItemsPage.toChannelPlaylistsResponse(): ChannelPlaylistsResponse = ChannelPlaylistsResponse( playlists = items.filterIsInstance().map { it.toPlaylistResultItem() }, nextpage = nextPage?.toCursor(), ) diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt index 37622ff0..c251943a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelResultItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ChannelResultItem import org.schabi.newpipe.extractor.channel.ChannelInfoItem -internal fun ChannelInfoItem.toChannelResultItem(): ChannelResultItem = ChannelResultItem( +fun ChannelInfoItem.toChannelResultItem(): ChannelResultItem = ChannelResultItem( id = url, name = name, url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt index af6f20a2..3641ce0a 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabExtraction.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.linkhandler.ChannelTabs import org.schabi.newpipe.extractor.search.filter.Filter import org.schabi.newpipe.extractor.search.filter.FilterItem -internal fun StreamingService.channelTabExtractor( +fun StreamingService.channelTabExtractor( url: String, channelId: String, tab: String, @@ -18,7 +18,7 @@ internal fun StreamingService.channelTabExtractor( return getChannelTabExtractor(linkHandler) } -internal fun StreamingService.toChannelTab(url: String, sort: String?): String? { +fun StreamingService.toChannelTab(url: String, sort: String?): String? { if (channelTabLHFactory == null) return null return url.toChannelTab(sort) } diff --git a/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt index 3abffd24..7b51f61b 100644 --- a/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelTabResolver.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.linkhandler.ChannelTabs -internal fun String.toChannelTab(sort: String?): String? { +fun String.toChannelTab(sort: String?): String? { if (contains("/shorts", ignoreCase = true)) return ChannelTabs.SHORTS if (contains("/streams", ignoreCase = true)) return ChannelTabs.LIVESTREAMS if (contains("/livestreams", ignoreCase = true)) return ChannelTabs.LIVESTREAMS @@ -10,7 +10,7 @@ internal fun String.toChannelTab(sort: String?): String? { return if (sort != null) ChannelTabs.VIDEOS else null } -internal fun String.toBaseChannelUrl(tab: String): String = substringBefore("/$tab") +fun String.toBaseChannelUrl(tab: String): String = substringBefore("/$tab") .substringBefore("/streams") .substringBefore('?') .substringBefore('#') diff --git a/src/main/kotlin/dev/typetype/server/services/CommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt b/server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/CustomAvatarUploadReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt index ec7a5407..64f679dd 100644 --- a/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DashManifestStreamCompatibility.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoStreamItem -internal fun isDashManifestVideoStream(stream: VideoStreamItem): Boolean { +fun isDashManifestVideoStream(stream: VideoStreamItem): Boolean { val codec = stream.codec ?: return false val height = stream.height.takeIf { it > 0 } ?: dashResolutionHeight(stream.resolution) return stream.url.isNotBlank() && codec.startsWith("avc1") && (height <= 0 || height <= 1080) diff --git a/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt diff --git a/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/DeArrowService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/DownloaderGatewayService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ExtractionErrorSanitizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt index b7c21e3e..6c806b75 100644 --- a/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException -internal suspend fun withExtractionRetry( +suspend fun withExtractionRetry( attempts: Int = 3, initialDelayMs: Long = 250, block: suspend () -> T, diff --git a/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt similarity index 72% rename from src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt index d98b6e2f..8b1315ba 100644 --- a/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/FavoriteItemFallback.kt @@ -2,5 +2,5 @@ package dev.typetype.server.services import dev.typetype.server.models.FavoriteItem -internal fun FavoriteItem.withYoutubeFallbackTitle(): FavoriteItem = +fun FavoriteItem.withYoutubeFallbackTitle(): FavoriteItem = if (title.isNotBlank()) this else copy(title = YoutubeTypeTypeMapper.titleForUrl(videoUrl)) diff --git a/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/FavoritesService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/FavoritesService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/GitHubIssueService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt index 766b240c..031de29e 100644 --- a/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryProgressMapper.kt @@ -9,7 +9,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal object HistoryProgressMapper { +object HistoryProgressMapper { fun toHistoryItems(userId: String, rows: List): List { val items = rows.map { it.toHistoryItem() } val savedProgress = savedProgressSeconds(userId, items.map { it.url }) diff --git a/src/main/kotlin/dev/typetype/server/services/HistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt index 428be477..8d2fea2b 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestCache.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.cache.CacheService -internal class HlsManifestCache(private val cache: CacheService) { +class HlsManifestCache(private val cache: CacheService) { suspend fun get(manifestUrl: String): String? = runCatching { cache.get(key(manifestUrl)) }.getOrNull() diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt index 4a3daa14..7d7878c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestRewrite.kt @@ -4,7 +4,7 @@ import java.net.URLEncoder import java.net.URI import java.nio.charset.StandardCharsets -internal fun isManifestUrl(url: String): Boolean { +fun isManifestUrl(url: String): Boolean { val uri = runCatching { URI(url) }.getOrNull() ?: return false if (uri.scheme != "http" && uri.scheme != "https") return false if (uri.path.contains("/file/seg.ts")) return false @@ -12,7 +12,7 @@ internal fun isManifestUrl(url: String): Boolean { return host == "manifest.googlevideo.com" || uri.path.endsWith(".m3u8", ignoreCase = true) } -internal fun rewriteYouTubeHlsManifest( +fun rewriteYouTubeHlsManifest( manifest: String, mapManifestUrl: (String) -> String = ::toHlsProxyUrl, ): String { @@ -27,7 +27,7 @@ internal fun rewriteYouTubeHlsManifest( } } -internal fun toHlsProxyUrl(url: String): String { +fun toHlsProxyUrl(url: String): String { val encoded = URLEncoder.encode(url, StandardCharsets.UTF_8) return if (isManifestUrl(url)) "hls-manifest?url=$encoded" else "../proxy?url=$encoded" } diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationApiArgs.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidatePool.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationContextualBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorCodec.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPageIndex.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorPayload.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationCursorState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecay.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDecision.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDelayedCreditBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceClass.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDeviceScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationDiscoveryAssembler.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEngagementSplitCalculator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationEventAnalyzer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExplorationQueryProvider.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationExploreBonus.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationFeedHistoryPenalty.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationJitter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLanguageGate.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationLiveTitleDetector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMetrics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixEngine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixRound.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMixer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationMomentumMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNextCursor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationNoopStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationOfflineEvaluator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPage.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPageDedup.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaDrift.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonaTracker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPersonalization.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPickState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPicker.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverDependencies.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolResolverState.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolRotation.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolWeights.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationProfile.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationQuotaPlanner.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationRelatedCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoredVideo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationScoring.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSearchCandidateFetcher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSemanticKey.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSessionIntentScorer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortProfileFallback.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortSubscriptionSource.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsAudit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsCandidateService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsClassifier.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDebugInfo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDeduplicator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsDefaults.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsFallback.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsHistorySignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsProfileFit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsQueryFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsRefresher.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSeenMemory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSignals.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSourceWeights.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationShortsSources.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalContext.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSignalProfileBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceBandit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationSourceTag.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationStateMachine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTaggedVideo.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTargetPlan.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTemporalBoost.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationThemeExtractor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTokenNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationTopicPairs.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationUserSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmup.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeRecommendationWarmupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt index a78aea45..3a244b6c 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HomeWarmupTracker.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal class HomeWarmupTracker( +class HomeWarmupTracker( private val throttleMs: Long, private val activeTtlMs: Long, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/InstanceService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/InstanceService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/InstanceService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/InternalHealthService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt b/server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt similarity index 61% rename from src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt index ff5a46d3..077fccc9 100644 --- a/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/LiveStreamTypes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import org.schabi.newpipe.extractor.stream.StreamType -internal data class StreamLiveMetadata( +data class StreamLiveMetadata( val streamType: String, val hlsUrl: String, val dashMpdUrl: String, @@ -12,9 +12,9 @@ internal data class StreamLiveMetadata( val hasLiveManifest: Boolean, ) -internal fun StreamType?.toApiStreamType(fallback: String = ""): String = this?.name?.lowercase() ?: fallback +fun StreamType?.toApiStreamType(fallback: String = ""): String = this?.name?.lowercase() ?: fallback -internal fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMpdUrl: String?): StreamLiveMetadata { +fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMpdUrl: String?): StreamLiveMetadata { val apiStreamType = streamType.toApiStreamType() val apiHlsUrl = hlsUrl?.takeIf { it.startsWith("http") } ?: "" val apiDashMpdUrl = dashMpdUrl?.takeIf { it.startsWith("http") } ?: "" @@ -29,8 +29,8 @@ internal fun streamLiveMetadata(streamType: StreamType?, hlsUrl: String?, dashMp ) } -internal fun String.isLiveStreamType(): Boolean = this == "live_stream" || this == "audio_live_stream" +fun String.isLiveStreamType(): Boolean = this == "live_stream" || this == "audio_live_stream" -internal fun String.isPostLiveStreamType(): Boolean = this == "post_live_stream" || this == "post_live_audio_stream" +fun String.isPostLiveStreamType(): Boolean = this == "post_live_stream" || this == "post_live_audio_stream" -internal fun String.isLiveContentType(): Boolean = isLiveStreamType() || isPostLiveStreamType() +fun String.isLiveContentType(): Boolean = isLiveStreamType() || isPostLiveStreamType() diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt index 578edcd2..58dd025b 100644 --- a/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestRepresentationIds.kt @@ -5,14 +5,14 @@ import dev.typetype.server.models.VideoStreamItem import org.schabi.newpipe.extractor.stream.AudioStream import org.schabi.newpipe.extractor.stream.VideoStream -internal fun VideoStreamItem.manifestRepresentationId(index: Int): String = "v-${itag.takeIf { it > 0 } ?: index}" +fun VideoStreamItem.manifestRepresentationId(index: Int): String = "v-${itag.takeIf { it > 0 } ?: index}" -internal fun AudioStreamItem.manifestRepresentationId(index: Int): String = +fun AudioStreamItem.manifestRepresentationId(index: Int): String = stableAudioRepresentationId(itag = itag, trackId = audioTrackId, index = index) -internal fun VideoStream.manifestRepresentationId(index: Int): String = "v-${getItag().takeIf { it > 0 } ?: index}" +fun VideoStream.manifestRepresentationId(index: Int): String = "v-${getItag().takeIf { it > 0 } ?: index}" -internal fun AudioStream.manifestRepresentationId(index: Int): String = +fun AudioStream.manifestRepresentationId(index: Int): String = stableAudioRepresentationId(itag = getItag(), trackId = getAudioTrackId(), index = index) private fun stableAudioRepresentationId(itag: Int, trackId: String?, index: Int): String { diff --git a/src/main/kotlin/dev/typetype/server/services/ManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt index 97b855b0..bdef81ff 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestBuilder.kt @@ -6,7 +6,7 @@ import org.schabi.newpipe.extractor.stream.AudioStream import org.schabi.newpipe.extractor.stream.DeliveryMethod import org.schabi.newpipe.extractor.stream.VideoStream -internal object NativeManifestBuilder { +object NativeManifestBuilder { fun build(videos: List, audios: List, duration: Long, preferredAudioTrackId: String?): String { val sb = StringBuilder() diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt similarity index 73% rename from src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt index 3b4e62d0..92203121 100644 --- a/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NativeManifestUtils.kt @@ -3,10 +3,10 @@ package dev.typetype.server.services import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal val REPRESENTATION_REGEX = Regex("""""") +val REPRESENTATION_REGEX = Regex("""""") private val MANIFEST_GOOGLEVIDEO_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/[^"<\s]+""") -internal fun rewriteManifestUrls(manifest: String): String = +fun rewriteManifestUrls(manifest: String): String = manifest.replace(MANIFEST_GOOGLEVIDEO_REGEX) { match -> val raw = match.value.replace("&", "&") val withPlaceholders = raw @@ -20,14 +20,14 @@ internal fun rewriteManifestUrls(manifest: String): String = .replace("TMPL_TIME", "\$Time\$") } -internal fun encodeUrl(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) +fun encodeUrl(url: String): String = URLEncoder.encode(url, StandardCharsets.UTF_8) -internal fun videoMimeType(codec: String): String = when { +fun videoMimeType(codec: String): String = when { codec.startsWith("vp9") || codec.startsWith("vp09") || codec.startsWith("av01") -> "video/webm" else -> "video/mp4" } -internal fun audioMimeType(codec: String): String = when { +fun audioMimeType(codec: String): String = when { codec.startsWith("opus") || codec.startsWith("vorbis") -> "audio/webm" else -> "audio/mp4" } diff --git a/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NicoNicoTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 8bb0c0f9..5f8b1a60 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -13,7 +13,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit -internal fun parseNicoCookie(fragment: String): String? { +fun parseNicoCookie(fragment: String): String? { val decoded = URLDecoder.decode(fragment, StandardCharsets.UTF_8) val cookieParam = decoded.split("&") .firstOrNull { it.startsWith("cookie=") } @@ -23,7 +23,7 @@ internal fun parseNicoCookie(fragment: String): String? { return cookieParam.substring(eqIdx + 1) } -internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: String? = null, proxyPath: String = "nicovideo"): String { +fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: String? = null, proxyPath: String = "nicovideo"): String { val base = URI(baseUrl) val uriAttr = Regex("""URI="([^"]+)"""") val bidSuffix = if (domandBid != null) "&domand_bid=${URLEncoder.encode(domandBid, StandardCharsets.UTF_8)}" else "" diff --git a/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/NotificationsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/NotificationsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 9b906a23..46c84692 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -12,15 +12,15 @@ import java.io.ByteArrayInputStream import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") +val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") private val CPN_TRACKING_PARAM_REGEX = Regex("[&?]cpn=[^&]*") private val PPPID_TRACKING_PARAM_REGEX = Regex("[&?]pppid=[^&]*") -internal fun stripTrackingParams(url: String): String = +fun stripTrackingParams(url: String): String = url.replace(CPN_TRACKING_PARAM_REGEX, "") .replace(PPPID_TRACKING_PARAM_REGEX, "") -internal fun rewriteHlsManifest(manifest: String): String = +fun rewriteHlsManifest(manifest: String): String = manifest.replace(GOOGLEVIDEO_URL_REGEX) { match -> "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } @@ -162,7 +162,7 @@ class OkHttpProxyService( } } -internal fun acceptsYoutubeThumbnailFallback(url: String, status: Int, contentType: String?): Boolean { +fun acceptsYoutubeThumbnailFallback(url: String, status: Int, contentType: String?): Boolean { val host = runCatching { java.net.URI(url).host.orEmpty() }.getOrDefault("") return status == 404 && host.endsWith("ytimg.com") && contentType?.startsWith("image/") == true } diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OkHttpYoutubeRemoteBrowserClient.kt diff --git a/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/OpenMojiProxyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt index dca441e0..5c2db6a3 100644 --- a/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PageSurrogate.kt @@ -8,7 +8,7 @@ import kotlinx.serialization.json.Json import org.schabi.newpipe.extractor.Page @Serializable -internal data class PageSurrogate( +data class PageSurrogate( val url: String?, val id: String?, val ids: List?, @@ -16,7 +16,7 @@ internal data class PageSurrogate( val body: String?, ) -internal fun Page.toCursor(): String { +fun Page.toCursor(): String { val surrogate = PageSurrogate( url = url, id = id, @@ -27,7 +27,7 @@ internal fun Page.toCursor(): String { return Base64.getEncoder().encodeToString(Json.encodeToString(surrogate).toByteArray()) } -internal fun String.toPage(): Page { +fun String.toPage(): Page { val json = String(Base64.getDecoder().decode(this)) val surrogate = Json.decodeFromString(json) return Page( diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupImporterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupLimits.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupReadModels.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupRestoreResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotItem.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSnapshotTimeMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteMeta.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupSqliteReader.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimeMode.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupTimelineNormalizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupUploadWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupValidators.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBackupZipExtractor.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeBulletCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt index 422b215d..648ebc1a 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeCommentService.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.comments.CommentsInfo import org.schabi.newpipe.extractor.comments.CommentsInfoItem -internal fun String.normalizeHttpSchema(): String = replace("httpss://", "https://") +fun String.normalizeHttpSchema(): String = replace("httpss://", "https://") class PipePipeCommentService : CommentService { diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt index 0d2579f7..7621ca58 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastEpisodesService.kt @@ -10,7 +10,7 @@ import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.playlist.PlaylistInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal class PipePipePodcastEpisodesService { +class PipePipePodcastEpisodesService { suspend fun getPodcastEpisodes(url: String, nextpage: String?): ExtractionResult = withContext(Dispatchers.IO) { diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipePublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index 85209166..23de1e04 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -31,7 +31,7 @@ private val ALL_SPONSOR_BLOCK_SETTINGS = SponsorBlockApiSettings().also { it.includeFillerCategory = true } -internal class PipePipeStreamService( +class PipePipeStreamService( private val cache: CacheService, private val subtitleService: YouTubeSubtitleService, private val bilibiliRelatedService: BilibiliRelatedService, @@ -119,5 +119,5 @@ internal class PipePipeStreamService( } } -internal suspend fun runPipePipeCall(block: () -> T): T = +suspend fun runPipePipeCall(block: () -> T): T = runInterruptible(context = Dispatchers.IO, block = block) diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PipePipeTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt index 0652c8c2..e29a0aa8 100644 --- a/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistResultItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.PlaylistResultItem import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem -internal fun PlaylistInfoItem.toPlaylistResultItem(): PlaylistResultItem = PlaylistResultItem( +fun PlaylistInfoItem.toPlaylistResultItem(): PlaylistResultItem = PlaylistResultItem( id = url ?: "", title = name ?: "", url = url ?: "", diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt index 1d07f770..b6a40874 100644 --- a/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt @@ -12,7 +12,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal fun playlistProgressByUrl(userId: String, videoUrls: List): Map { +fun playlistProgressByUrl(userId: String, videoUrls: List): Map { val urls = videoUrls.distinct().filter { it.isNotBlank() } if (urls.isEmpty()) return emptyMap() return ProgressTable.selectAll() @@ -26,7 +26,7 @@ internal fun playlistProgressByUrl(userId: String, videoUrls: List): Map } } -internal fun ResultRow.toPlaylistSummary(videoCount: Int): PlaylistItem = PlaylistItem( +fun ResultRow.toPlaylistSummary(videoCount: Int): PlaylistItem = PlaylistItem( id = this[PlaylistsTable.id], name = this[PlaylistsTable.name], description = this[PlaylistsTable.description], @@ -34,7 +34,7 @@ internal fun ResultRow.toPlaylistSummary(videoCount: Int): PlaylistItem = Playli createdAt = this[PlaylistsTable.createdAt], ) -internal fun ResultRow.toPlaylistVideoItem(progressByUrl: Map): PlaylistVideoItem { +fun ResultRow.toPlaylistVideoItem(progressByUrl: Map): PlaylistVideoItem { val url = this[PlaylistVideosTable.url] val duration = this[PlaylistVideosTable.duration] val progress = progressByUrl[url] diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt similarity index 79% rename from src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt index 0ca1ba28..e6b918a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastItemMappers.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.PodcastItem import org.schabi.newpipe.extractor.playlist.PlaylistInfo import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem -internal fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( +fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -14,7 +14,7 @@ internal fun PlaylistInfoItem.toPodcastItem(): PodcastItem = PodcastItem( playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( +fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -24,7 +24,7 @@ internal fun PlaylistInfo.toPodcastItem(): PodcastItem = PodcastItem( playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun emptyPodcastItem(url: String): PodcastItem = PodcastItem( +fun emptyPodcastItem(url: String): PodcastItem = PodcastItem( id = url, title = "", url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/PodcastService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PodcastService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PodcastService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PodcastService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PresenceKeyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PresenceService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PresenceService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt index 1ac9340b..2d49e1da 100644 --- a/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PresenceTokenCodec.kt @@ -4,8 +4,8 @@ import java.security.MessageDigest import java.security.SecureRandom import java.util.Base64 -internal object PresenceTokenCodec { - internal const val PREFIX = "ttp1_" +object PresenceTokenCodec { + const val PREFIX = "ttp1_" private val random = SecureRandom() fun issue(): String { diff --git a/src/main/kotlin/dev/typetype/server/services/ProgressService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProgressService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ProgressService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProgressService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt index e7f2a361..cdb29c5d 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderHlsManifestRewrite.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import java.net.URI -internal suspend fun rewriteProviderHlsManifest( +suspend fun rewriteProviderHlsManifest( manifest: String, baseUrl: String, mapUrl: suspend (String) -> String, diff --git a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt index 04d46dff..07f54644 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProviderMediaHandleService.kt @@ -19,13 +19,13 @@ import java.security.MessageDigest import java.util.Base64 import java.util.concurrent.ConcurrentHashMap -internal enum class ProviderMediaType { +enum class ProviderMediaType { BILIBILI, NICONICO, } @Serializable -internal data class ProviderMediaTarget( +data class ProviderMediaTarget( val url: String, val domandBid: String? = null, ) @@ -37,7 +37,7 @@ class ProviderMediaHandleService( private val writeLimiter = Semaphore(MAX_CONCURRENT_CACHE_WRITES) private val inFlightWrites = ConcurrentHashMap>() - internal suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { + suspend fun materialize(response: StreamResponse, provider: ProviderMediaType): StreamResponse = coroutineScope { val hls = async { response.hlsUrl.handleIfRemote(provider) } val dash = async { response.dashMpdUrl.handleIfRemote(provider) } val video = async { response.videoStreams.withHandledVideoUrls(provider) } @@ -52,7 +52,7 @@ class ProviderMediaHandleService( ) } - internal suspend fun createPath(rawUrl: String, domandBid: String? = null): String { + suspend fun createPath(rawUrl: String, domandBid: String? = null): String { val (cleanUrl, fragment) = rawUrl.splitFragment() val resolvedBid = domandBid?.takeIf { it.isNotBlank() } ?: fragment.takeIf { it.isNotBlank() }?.let(::parseNicoCookie) @@ -81,14 +81,14 @@ class ProviderMediaHandleService( } } - internal suspend fun resolve(handle: String): ProviderMediaTarget? { + suspend fun resolve(handle: String): ProviderMediaTarget? { if (!HANDLE_PATTERN.matches(handle)) return null return cache.get(cacheKey(handle))?.let { encoded -> runCatching { CacheJson.decodeFromString(encoded) }.getOrNull() } } - internal fun relativeManifestPath(path: String): String = + fun relativeManifestPath(path: String): String = "../media/${path.substringAfterLast('/')}" private suspend fun String.handleIfRemote(provider: ProviderMediaType): String { diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt index 4f27c759..350eea33 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt @@ -12,7 +12,7 @@ import java.net.Proxy import java.net.UnknownHostException import java.util.concurrent.ConcurrentHashMap -internal class ProxyHttpExecutor( +class ProxyHttpExecutor( client: OkHttpClient, private val maxRedirects: Int = 5, ) { @@ -59,7 +59,7 @@ internal class ProxyHttpExecutor( } } -internal class ValidatingProxyDns(private val delegate: Dns) : Dns { +class ValidatingProxyDns(private val delegate: Dns) : Dns { private val trustedTransportHosts = ConcurrentHashMap.newKeySet() override fun lookup(hostname: String): List { diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/ProxyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt index 1286bc5e..5f677b1c 100644 --- a/src/main/kotlin/dev/typetype/server/services/ProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ProxyService.kt @@ -7,7 +7,7 @@ interface ProxyService { suspend fun pipe(url: String, rangeHeader: String?, domandBid: String? = null): ExtractionResult } -internal interface ProviderMediaAwareProxyService { +interface ProviderMediaAwareProxyService { suspend fun pipeProviderMedia( url: String, rangeHeader: String?, diff --git a/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt index 7513ab18..5e139ca8 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicCachePolicy.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal object PublicCachePolicy { +object PublicCachePolicy { fun trendingTtl(serviceId: Int): Long = when (serviceId) { BILIBILI_SERVICE_ID, NICONICO_SERVICE_ID -> 600L YOUTUBE_SERVICE_ID -> 1_800L diff --git a/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt index 8771fc29..24f4d574 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicExtractionCache.kt @@ -6,7 +6,7 @@ import dev.typetype.server.models.ExtractionResult import kotlinx.serialization.KSerializer import org.slf4j.LoggerFactory -internal object PublicExtractionCache { +object PublicExtractionCache { private val logger = LoggerFactory.getLogger(PublicExtractionCache::class.java) suspend fun getOrLoad( diff --git a/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt similarity index 74% rename from src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt index 4b7d2c17..4ff44bb4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistItemMappers.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.PublicPlaylistItem import org.schabi.newpipe.extractor.playlist.PlaylistInfo -internal fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPlaylistItem( +fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPlaylistItem( id = url ?: "", title = name ?: "", url = url ?: "", @@ -13,7 +13,7 @@ internal fun PlaylistInfo.toPublicPlaylistItem(): PublicPlaylistItem = PublicPla playlistType = playlistType?.name?.lowercase() ?: "", ) -internal fun emptyPublicPlaylistItem(url: String): PublicPlaylistItem = PublicPlaylistItem( +fun emptyPublicPlaylistItem(url: String): PublicPlaylistItem = PublicPlaylistItem( id = url, title = "", url = url, diff --git a/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt index 3233cad5..a98c34a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublishedAtMapper.kt @@ -1,5 +1,5 @@ package dev.typetype.server.services -internal object PublishedAtMapper { +object PublishedAtMapper { fun fromUploaded(uploaded: Long): Long? = uploaded.takeIf { it > 0L } } diff --git a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt index 5ad17817..f7a67f17 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushDeviceRegistry.kt @@ -14,7 +14,7 @@ import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class PushDeviceRegistry( +class PushDeviceRegistry( private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), private val maxDevicesPerAccount: Int = DEFAULT_MAX_DEVICES, ) { @@ -135,9 +135,9 @@ internal class PushDeviceRegistry( } } -internal data class PushDevice(val id: String, val userId: String, val endpoint: String) +data class PushDevice(val id: String, val userId: String, val endpoint: String) -internal sealed interface DeviceRegistrationResult { +sealed interface DeviceRegistrationResult { data class Success(val response: PushDeviceRegistrationResponse) : DeviceRegistrationResult data class Invalid(val reason: String) : DeviceRegistrationResult data object UnsupportedPlatform : DeviceRegistrationResult diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt index 5d5315e3..921444fd 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationDeliveryStore.kt @@ -13,7 +13,7 @@ import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update -internal class PushNotificationDeliveryStore( +class PushNotificationDeliveryStore( private val clock: () -> Long, ) { fun markSeen(userId: String, candidate: PushCandidate, now: Long): Boolean = @@ -97,7 +97,7 @@ internal class PushNotificationDeliveryStore( } } -internal object PushNotificationDeliveryStatus { +object PushNotificationDeliveryStatus { const val PENDING = "pending" const val DELIVERED = "delivered" const val INVALID = "invalid" diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt index e82a5285..62bd2778 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationScheduler.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -internal class PushNotificationScheduler( +class PushNotificationScheduler( private val service: PushNotificationService, private val intervalMs: Long = configuredIntervalMs(), private val initialDelayMs: Long = 10_000L, diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt index dd0ad996..fe0b4696 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationService.kt @@ -13,7 +13,7 @@ import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.insertIgnore import org.jetbrains.exposed.v1.jdbc.selectAll -internal class PushNotificationService( +class PushNotificationService( private val subscriptionsService: SubscriptionsService, private val subscriptionFeedService: SubscriptionFeedService, private val preferenceService: ChannelNotificationPreferenceService, @@ -76,7 +76,7 @@ internal class PushNotificationService( } } - internal suspend fun pollUser(userId: String) { + suspend fun pollUser(userId: String) { val subscriptions = subscriptionsService.getAll(userId) if (subscriptions.isEmpty()) return val devices = deviceRegistry.activeDevices(userId, clock()) @@ -153,7 +153,7 @@ internal class PushNotificationService( } } -internal data class PushCandidate( +data class PushCandidate( val serviceId: Int, val channelId: String, val videoId: String, @@ -161,4 +161,4 @@ internal data class PushCandidate( val video: VideoItem, ) -internal data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) +data class DeliveryWork(val eventId: String, val device: PushDevice, val payload: String) diff --git a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt index fc1d548e..753f8f54 100644 --- a/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PushNotificationSupport.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.UnifiedPushNotificationPayload import dev.typetype.server.models.VideoItem import java.security.MessageDigest -internal object PushNotificationSupport { +object PushNotificationSupport { fun candidates( video: VideoItem, sources: Map>, diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationEventService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryEntry.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedHistoryService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationFeedbackSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestBatchUpdater.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestProfileService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationInterestWeight.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCatalog.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingCreativeTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEducationTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingEntertainmentTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingGamingTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingHealthFitnessTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingLifestyleTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingMusicTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingNewsTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingPreferencesStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingScienceNatureTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingSeedService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingStateStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTechnologyTopics.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationOnboardingTopicGroups.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationPrivacyService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt index 63bcdbac..059abc73 100644 --- a/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationServiceId.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import java.net.URI -internal fun recommendationServiceId(url: String): Int { +fun recommendationServiceId(url: String): Int { val host = runCatching { URI(url).host.orEmpty().lowercase() }.getOrDefault("") return when { host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> BILIBILI_SERVICE_ID diff --git a/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RecommendationTopicTokenizer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt index bb546633..ea85822d 100644 --- a/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RelatedItemMappers.kt @@ -11,16 +11,16 @@ private val JAPANESE_DATE_REGEX = Regex("""\d{4}年\d{2}月\d{2}日 \d{2}[：:]\ private fun String.extractJapaneseDate(): String = JAPANESE_DATE_REGEX.find(this)?.value ?: this -internal fun String?.toAbsoluteUrl(): String = if (this?.startsWith("http") == true) this else "" +fun String?.toAbsoluteUrl(): String = if (this?.startsWith("http") == true) this else "" -internal fun SponsorBlockSegment.toSegmentItem(): SponsorBlockSegmentItem = SponsorBlockSegmentItem( +fun SponsorBlockSegment.toSegmentItem(): SponsorBlockSegmentItem = SponsorBlockSegmentItem( startTime = startTime, endTime = endTime, category = category.apiName, action = action.apiName, ) -internal fun List.toSponsorBlockSegments(): Array = +fun List.toSponsorBlockSegments(): Array = mapNotNull { item -> runCatching { SponsorBlockSegment( @@ -34,7 +34,7 @@ internal fun List.toSponsorBlockSegments(): Array Response, private val request: Request, initialResponse: Response, diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt index 169281fa..4d6defd6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -12,7 +12,7 @@ import java.time.format.DateTimeFormatter import javax.xml.stream.XMLOutputFactory import javax.xml.stream.XMLStreamWriter -internal object RssDocumentRenderer { +object RssDocumentRenderer { fun render( feed: RssFeedItem, videos: List, diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt index 76816a96..0bdccb9f 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt @@ -10,7 +10,7 @@ import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.selectAll -internal class RssFeedAdminRepository { +class RssFeedAdminRepository { suspend fun list(page: Int, limit: Int): Pair, Long> = DatabaseFactory.query { val total = RssFeedsTable.selectAll().count() val rows = RssFeedsTable.selectAll() diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt index 868f6a30..08eb7ab7 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt @@ -8,7 +8,7 @@ import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.util.UUID -class RssFeedManagementService internal constructor( +class RssFeedManagementService public constructor( private val settings: AdminSettingsService, private val subscriptions: SubscriptionsService, private val repository: RssFeedRepository = RssFeedRepository(), diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt index 18fc0b95..818c3e33 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.VideoItem import java.security.MessageDigest import java.util.HexFormat -class RssFeedReaderService internal constructor( +class RssFeedReaderService public constructor( private val settings: AdminSettingsService, private val subscriptionFeed: SubscriptionFeedService, private val blocked: BlockedService, diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt index d8fab751..e5fc87dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt @@ -17,9 +17,9 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager import org.jetbrains.exposed.v1.jdbc.update -internal data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) +data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) -internal class RssFeedRepository { +class RssFeedRepository { suspend fun list(userId: String): List = DatabaseFactory.query { val rows = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt similarity index 89% rename from src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt index 55421556..219256a6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt @@ -4,10 +4,10 @@ import dev.typetype.server.db.tables.RssFeedsTable import dev.typetype.server.models.RssFeedItem import org.jetbrains.exposed.v1.core.ResultRow -internal fun ResultRow.toStoredFeed(): StoredRssFeed = +fun ResultRow.toStoredFeed(): StoredRssFeed = toStoredFeed(loadRssFeedSelections(listOf(this[RssFeedsTable.id]))) -internal fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { +fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { val id = this[RssFeedsTable.id] return StoredRssFeed( item = RssFeedItem( diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt index 563dda78..a947c75d 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt @@ -4,7 +4,7 @@ import java.security.MessageDigest import java.security.SecureRandom import java.util.Base64 -internal class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { +class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { fun create(): String { val bytes = ByteArray(32) random.nextBytes(bytes) diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt similarity index 88% rename from src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt index 17343952..af575ab6 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt @@ -5,12 +5,12 @@ import dev.typetype.server.db.tables.RssFeedServicesTable import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.jdbc.selectAll -internal data class RssFeedSelections( +data class RssFeedSelections( val channels: Map>, val services: Map>, ) -internal fun loadRssFeedSelections(feedIds: List): RssFeedSelections { +fun loadRssFeedSelections(feedIds: List): RssFeedSelections { if (feedIds.isEmpty()) return RssFeedSelections(emptyMap(), emptyMap()) val channels = RssFeedChannelsTable.selectAll() .where { RssFeedChannelsTable.feedId inList feedIds } diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt index 87e81317..c27af1c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger -internal class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { +class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { private val windows = ConcurrentHashMap() private val acquisitions = AtomicInteger() diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt index 047eaaab..7b59c322 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem import java.net.URI -internal object RssVideoMetadata { +object RssVideoMetadata { fun serviceId(video: VideoItem): Int { val host = runCatching { URI(video.url).host.orEmpty().lowercase() }.getOrDefault("") return when { diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt index a7919c07..4db8b756 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.RssFeedItem import dev.typetype.server.models.VideoItem -internal object RssVideoTypeFilter { +object RssVideoTypeFilter { fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { video.isLive -> feed.includeLive video.isUpcomingAt(now) -> feed.includeUpcoming diff --git a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt index 3760acf4..4c0e828c 100644 --- a/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SavedPlaylistWrite.kt @@ -4,7 +4,7 @@ import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.models.PublicPlaylistItem import org.jetbrains.exposed.v1.core.statements.UpdateBuilder -internal fun UpdateBuilder<*>.writePlaylist(playlist: PublicPlaylistItem, savedAt: Long): Unit { +fun UpdateBuilder<*>.writePlaylist(playlist: PublicPlaylistItem, savedAt: Long): Unit { this[SavedPlaylistsTable.publicPlaylistId] = playlist.id this[SavedPlaylistsTable.url] = playlist.url this[SavedPlaylistsTable.title] = playlist.title diff --git a/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt similarity index 76% rename from src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt index bfe703ac..59ed00e8 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchContentKind.kt @@ -3,20 +3,20 @@ package dev.typetype.server.services import dev.typetype.server.models.SearchPageResponse import org.schabi.newpipe.extractor.search.filter.FilterItem -internal enum class SearchContentKind { +enum class SearchContentKind { All, Videos, Playlists, Channels, } -internal fun String?.toSearchContentKind(fallbackFilters: List): SearchContentKind = this +fun String?.toSearchContentKind(fallbackFilters: List): SearchContentKind = this ?.substringAfterLast('|') ?.toSearchContentKind() ?.takeUnless { it == SearchContentKind.All } ?: fallbackFilters.toSearchContentKind() -internal fun List.toSearchContentKind(): SearchContentKind = firstOrNull() +fun List.toSearchContentKind(): SearchContentKind = firstOrNull() ?.name ?.lowercase() ?.toSearchContentKind() @@ -29,7 +29,7 @@ private fun String.toSearchContentKind(): SearchContentKind = when (lowercase()) else -> SearchContentKind.All } -internal fun SearchPageResponse.filteredBy(kind: SearchContentKind): SearchPageResponse = when (kind) { +fun SearchPageResponse.filteredBy(kind: SearchContentKind): SearchPageResponse = when (kind) { SearchContentKind.All -> this SearchContentKind.Videos -> copy(playlists = emptyList(), channels = emptyList()) SearchContentKind.Playlists -> copy(items = emptyList(), channels = emptyList()) diff --git a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt similarity index 82% rename from src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt index 80223e8e..1cdd9db1 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchFilterMappers.kt @@ -6,22 +6,22 @@ import org.schabi.newpipe.extractor.search.filter.Filter import org.schabi.newpipe.extractor.search.filter.FilterGroup import org.schabi.newpipe.extractor.search.filter.FilterItem -internal sealed interface SearchFilterResolution { +sealed interface SearchFilterResolution { data class Valid(val items: List) : SearchFilterResolution data class Invalid(val message: String) : SearchFilterResolution } -internal fun Filter?.toSearchContentFilterOptions(): List = entries() +fun Filter?.toSearchContentFilterOptions(): List = entries() .mapIndexed { index, entry -> entry.toOption(isDefault = index == 0, includeGroup = true) } -internal fun Filter?.toSearchFilterOptions(): List = this?.filterGroups +fun Filter?.toSearchFilterOptions(): List = this?.filterGroups ?.flatMap { group -> group.filterItems.mapIndexed { index, item -> FilterEntry(group, item).toOption(group.onlyOneCheckable && index == 0, includeGroup = true) } }.orEmpty() -internal fun Filter?.toSearchFilterGroups(): List = this?.filterGroups +fun Filter?.toSearchFilterGroups(): List = this?.filterGroups ?.map { group -> SearchFilterGroup( key = "${group.groupName.orEmpty()}|${group.identifier}", @@ -33,11 +33,11 @@ internal fun Filter?.toSearchFilterGroups(): List = this?.fil ) }.orEmpty() -internal fun Filter?.findSearchFilter(value: String?): List = value?.let { raw -> +fun Filter?.findSearchFilter(value: String?): List = value?.let { raw -> entries().firstOrNull { it.value == raw }?.let { listOf(it.item) } } ?: emptyList() -internal fun Filter?.resolveSearchFilters(values: List): SearchFilterResolution { +fun Filter?.resolveSearchFilters(values: List): SearchFilterResolution { if (values.isEmpty()) return SearchFilterResolution.Valid(emptyList()) val requested = values.toSet() val selected = entries().filter { it.value in requested } @@ -52,12 +52,12 @@ internal fun Filter?.resolveSearchFilters(values: List): SearchFilterRes return SearchFilterResolution.Valid(selected.map(FilterEntry::item)) } -internal fun Filter?.defaultSearchFilters(): List = this?.filterGroups +fun Filter?.defaultSearchFilters(): List = this?.filterGroups ?.filter(FilterGroup::onlyOneCheckable) ?.mapNotNull { it.filterItems.firstOrNull() } .orEmpty() -internal fun Filter?.defaultSearchFilter(): List = this?.filterGroups +fun Filter?.defaultSearchFilter(): List = this?.filterGroups ?.firstOrNull() ?.filterItems ?.firstOrNull() diff --git a/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchHistoryService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt index efd2feeb..847967aa 100644 --- a/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SearchPageMappers.kt @@ -8,7 +8,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem import org.schabi.newpipe.extractor.search.SearchInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem -internal fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( +fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( items = relatedItems.filterIsInstance().map { it.toVideoItem() }, nextpage = nextPage?.toCursor(), searchSuggestion = searchSuggestion?.takeIf { it.isNotBlank() }, @@ -17,7 +17,7 @@ internal fun SearchInfo.toSearchPageResponse(): SearchPageResponse = SearchPageR channels = relatedItems.filterIsInstance().map { it.toChannelResultItem() }, ) -internal fun InfoItemsPage.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( +fun InfoItemsPage.toSearchPageResponse(): SearchPageResponse = SearchPageResponse( items = items.filterIsInstance().map { it.toVideoItem() }, nextpage = nextPage?.toCursor(), searchSuggestion = null, diff --git a/src/main/kotlin/dev/typetype/server/services/SearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/ServiceIds.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/ServiceIds.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SettingsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index d609931b..8fc51f21 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,7 +35,7 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } - internal suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { + suspend fun subscriptionFeedVisibility(userId: String): SubscriptionFeedVisibility = DatabaseFactory.query { SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { SubscriptionFeedVisibility( hideLiveStreams = it[SettingsTable.hideSubscriptionLiveStreams], @@ -54,7 +54,7 @@ class SettingsService { } } -internal data class SubscriptionFeedVisibility( +data class SubscriptionFeedVisibility( val hideLiveStreams: Boolean = false, val hideMembersOnlyContent: Boolean = false, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestCookie.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestToken.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StoryboardProxyDetector.kt diff --git a/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamAudioContractResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt similarity index 94% rename from src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt index 69cd3304..f8d757f4 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -11,7 +11,7 @@ private const val BILIBILI_SIGNED_STREAM_MAX_TTL_SECONDS = 3_600L private const val SIGNED_STREAM_DEADLINE_SAFETY_SECONDS = 300L private const val MIN_CACHEABLE_STREAM_TTL_SECONDS = 60L -internal fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { +fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { val deadline = signedMediaUrls().mapNotNull { it.bilibiliDeadline() }.minOrNull() ?: return stableMetadataTtlSeconds() val ttl = deadline - nowEpochSeconds - SIGNED_STREAM_DEADLINE_SAFETY_SECONDS diff --git a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index 62b2cfdb..ac1cc565 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -12,7 +12,7 @@ import org.schabi.newpipe.extractor.exceptions.ReCaptchaException import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException -internal object StreamExtractionErrorMapper { +object StreamExtractionErrorMapper { const val MEMBERS_ONLY_FALLBACK = "This video is only available for members" const val PAID_CONTENT_FALLBACK = "This video is a paid video" const val GEOGRAPHIC_RESTRICTION_CODE = "geographic_restriction" diff --git a/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt index 7ed667c6..5ebe5add 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamInfoMappers.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem import org.schabi.newpipe.extractor.stream.SubtitlesStream -internal fun StreamInfo.toStreamResponse(): StreamResponse { +fun StreamInfo.toStreamResponse(): StreamResponse { val uploaded = uploadDate?.offsetDateTime()?.toInstant()?.toEpochMilli() ?: -1L val liveMetadata = streamLiveMetadata(streamType, hlsUrl, dashMpdUrl) return StreamResponse( @@ -57,7 +57,7 @@ internal fun StreamInfo.toStreamResponse(): StreamResponse { ) } -internal fun SubtitlesStream.toSubtitleItem(): SubtitleItem = SubtitleItem( +fun SubtitlesStream.toSubtitleItem(): SubtitleItem = SubtitleItem( url = getContent() ?: "", mimeType = getFormat()?.getMimeType() ?: "", languageTag = getLanguageTag() ?: "", diff --git a/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt index bc38188d..07262bca 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamMediaItemMappers.kt @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.stream.VideoStream import java.net.URLEncoder import java.nio.charset.StandardCharsets -internal fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean): VideoStreamItem { +fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean): VideoStreamItem { val method = deliveryMethodName() return VideoStreamItem( url = playableUrl(method), @@ -32,7 +32,7 @@ internal fun VideoStream.toVideoStreamItem(videoId: String, isVideoOnly: Boolean ) } -internal fun AudioStream.toAudioStreamItem(videoId: String): AudioStreamItem { +fun AudioStream.toAudioStreamItem(videoId: String): AudioStreamItem { val method = deliveryMethodName() return AudioStreamItem( url = playableUrl(method), @@ -85,10 +85,10 @@ private fun audioSessionUrl(videoId: String, itag: Int, trackId: String?): Strin return "/sabr/session/$videoId?audioItag=$itag$track" } -internal fun VideoStreamItem.isSupportedPlaybackStream(): Boolean = +fun VideoStreamItem.isSupportedPlaybackStream(): Boolean = deliveryMethod != "sabr" || isSupportedSabrVideo() -internal fun AudioStreamItem.isSupportedPlaybackStream(): Boolean = +fun AudioStreamItem.isSupportedPlaybackStream(): Boolean = deliveryMethod != "sabr" || isSupportedSabrAudio() private fun VideoStreamItem.isSupportedSabrVideo(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt similarity index 80% rename from src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt index a89f3b7d..ad7b561f 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamTimelineMappers.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.StreamSegmentItem import org.schabi.newpipe.extractor.stream.Frameset import org.schabi.newpipe.extractor.stream.StreamSegment -internal fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( +fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( urls = getUrls() ?: emptyList(), frameWidth = getFrameWidth(), frameHeight = getFrameHeight(), @@ -15,7 +15,7 @@ internal fun Frameset.toPreviewFrameItem(): PreviewFrameItem = PreviewFrameItem( framesPerPageY = getFramesPerPageY(), ) -internal fun StreamSegment.toStreamSegmentItem(): StreamSegmentItem = StreamSegmentItem( +fun StreamSegment.toStreamSegmentItem(): StreamSegmentItem = StreamSegmentItem( title = getTitle() ?: "", startTimeSeconds = getStartTimeSeconds(), channelName = getChannelName(), diff --git a/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt index 85643b1f..5050fc30 100644 --- a/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamYouTubeSubtitleResolver.kt @@ -8,11 +8,11 @@ import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -internal fun interface YouTubeSubtitleTrackResolver { +fun interface YouTubeSubtitleTrackResolver { suspend fun resolve(selection: YouTubeSubtitleSelection): YouTubeSubtitleResolution } -internal class StreamYouTubeSubtitleResolver( +class StreamYouTubeSubtitleResolver( private val streamService: StreamService, private val fetchInventory: suspend (String) -> YouTubeSubtitleInventoryResult, ) : YouTubeSubtitleTrackResolver { @@ -57,7 +57,7 @@ private suspend fun StreamService.streamForSubtitle(videoId: String): StreamResp private fun StreamResponse.isLiveSubtitleContent(): Boolean = isLive || isLiveContent -internal fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelection): Boolean { +fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelection): Boolean { val parsedUrl = url.toHttpUrlOrNull() val sourceLanguage = parsedUrl?.queryParameter("lang") ?: languageTag val requestedSource = selection.sourceLanguage ?: selection.language @@ -70,7 +70,7 @@ internal fun SubtitleItem.matchesYouTubeSubtitle(selection: YouTubeSubtitleSelec return selection.trackName == null || parsedUrl?.queryParameter("name") == selection.trackName } -internal fun SubtitleItem.contentForYouTubeSubtitle(selection: YouTubeSubtitleSelection): String? { +fun SubtitleItem.contentForYouTubeSubtitle(selection: YouTubeSubtitleSelection): String? { val parsedUrl = url.toHttpUrlOrNull()?.takeIf { isYouTubeTimedTextUrl(it.toString()) } ?: return null return parsedUrl.newBuilder() .setQueryParameter("fmt", selection.format.value) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt similarity index 86% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt index e29e0beb..33863129 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedAvailability.kt @@ -7,7 +7,7 @@ data class SubscriptionFeedAvailability( val available: Boolean, ) -internal data class SubscriptionFeedWithSources( +data class SubscriptionFeedWithSources( val videos: List, val available: Boolean, val sourceChannelUrls: Map>, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index 95beed0d..dbe8154d 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -11,7 +11,7 @@ import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withTimeoutOrNull import java.net.URI -internal class SubscriptionFeedBuilder(private val channelService: ChannelService) { +class SubscriptionFeedBuilder(private val channelService: ChannelService) { private val semaphore = Semaphore(MAX_CONCURRENT_FETCHES) suspend fun build(subscriptions: List): SubscriptionFeedBuildResult = coroutineScope { @@ -122,7 +122,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } } -internal data class SubscriptionFeedBuildResult( +data class SubscriptionFeedBuildResult( val videos: List, val sourceChannelUrls: Map>, val successfulSources: Int, diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheInvalidator.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedCacheKeys.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt similarity index 92% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt index 89f51501..786c5cef 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedOrderer.kt @@ -2,7 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.models.VideoItem -internal class SubscriptionFeedOrderer { +class SubscriptionFeedOrderer { fun order( videos: List, previous: SubscriptionFeedSnapshot?, @@ -45,10 +45,10 @@ internal class SubscriptionFeedOrderer { } } -internal data class SubscriptionFeedOrdering( +data class SubscriptionFeedOrdering( val videos: List, val livePromotedAt: Map, ) -internal fun VideoItem.subscriptionFeedKey(): String = +fun VideoItem.subscriptionFeedKey(): String = url.ifBlank { id.ifBlank { "$uploaderUrl|$title" } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt index b8bc1737..3d2e8ac5 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSelectionStore.kt @@ -5,7 +5,7 @@ import dev.typetype.server.cache.CacheService import kotlinx.serialization.Serializable import java.security.MessageDigest -internal class SubscriptionFeedSelectionStore( +class SubscriptionFeedSelectionStore( private val cache: CacheService, private val subscriptions: SubscriptionsService, ) { @@ -98,7 +98,7 @@ private data class SubscriptionFeedSelectionIdentity( val channelUrls: List, ) -internal data class SubscriptionFeedSelectionSnapshot( +data class SubscriptionFeedSelectionSnapshot( val token: String?, val channelUrls: Set?, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 019bf267..81b661ab 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -35,7 +35,7 @@ class SubscriptionFeedService( private val refreshJobs = ConcurrentHashMap() private val generation = AtomicLong(clock()) - internal suspend fun getPage( + suspend fun getPage( userId: String, page: Int, limit: Int, @@ -124,7 +124,7 @@ class SubscriptionFeedService( ) } - internal suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = + suspend fun getAllWithSources(userId: String): SubscriptionFeedWithSources = getAllWithAvailability(userId).let { SubscriptionFeedWithSources(it.videos, it.available, store.current(userId)?.sourceChannelUrls.orEmpty()) } suspend fun getCachedFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse? { @@ -132,7 +132,7 @@ class SubscriptionFeedService( return snapshot.page(page * limit, limit, isRefreshing(userId)) } - internal suspend fun getCachedAll(userId: String): List? = + suspend fun getCachedAll(userId: String): List? = store.current(userId)?.videos suspend fun invalidate(userId: String) { @@ -141,7 +141,7 @@ class SubscriptionFeedService( scheduleRefresh(userId, currentRequestId()) } - internal suspend fun awaitRefresh(userId: String) { + suspend fun awaitRefresh(userId: String) { while (true) { val jobs = listOfNotNull(refreshJobs[userId]) if (jobs.isEmpty()) return @@ -149,7 +149,7 @@ class SubscriptionFeedService( } } - internal fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true + fun isRefreshing(userId: String): Boolean = refreshJobs[userId]?.isActive == true fun close() { refreshJobs.values.forEach(Job::cancel) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index 436d7948..3ce28437 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -7,7 +7,7 @@ import kotlinx.serialization.Serializable import java.util.Base64 @Serializable -internal data class SubscriptionFeedSnapshot( +data class SubscriptionFeedSnapshot( val generation: Long, val generatedAt: Long, val stale: Boolean, @@ -27,7 +27,7 @@ private data class SubscriptionFeedCursor( val selectionToken: String? = null, ) -internal object SubscriptionFeedCursorCodec { +object SubscriptionFeedCursorCodec { fun encode( generation: Long, offset: Int, @@ -70,7 +70,7 @@ internal object SubscriptionFeedCursorCodec { private val SELECTION_TOKEN = Regex("[0-9a-f]{64}") } -internal data class SubscriptionFeedCursorState( +data class SubscriptionFeedCursorState( val generation: Long, val offset: Int, val limit: Int, @@ -80,10 +80,10 @@ internal data class SubscriptionFeedCursorState( val selectionToken: String?, ) -internal fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = +fun SubscriptionFeedSnapshot.hasCompleteSourceAttribution(): Boolean = videos.all { sourceChannelUrls.containsKey(it.subscriptionFeedKey()) } -internal fun SubscriptionFeedSnapshot.page( +fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, @@ -138,7 +138,7 @@ private fun SubscriptionFeedSnapshot.projectedVideos( } } -internal sealed interface SubscriptionFeedPageResult { +sealed interface SubscriptionFeedPageResult { data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult data object InvalidCursor : SubscriptionFeedPageResult diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt index 365e4308..a86a2a23 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshotStore.kt @@ -5,7 +5,7 @@ import dev.typetype.server.cache.CacheService import dev.typetype.server.models.VideoItem import kotlinx.serialization.builtins.ListSerializer -internal class SubscriptionFeedSnapshotStore( +class SubscriptionFeedSnapshotStore( private val cache: CacheService, private val clock: () -> Long, ) { diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsBlendService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsFeedService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionShortsSignalService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/SuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/SuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/TrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/TrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt index 4b59038e..4e1ce959 100644 --- a/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushEndpointValidator.kt @@ -4,7 +4,7 @@ import java.net.InetAddress import java.net.URI import java.net.UnknownHostException -internal class UnifiedPushEndpointValidator( +class UnifiedPushEndpointValidator( private val resolver: (String) -> Array = InetAddress::getAllByName, ) { fun validate(raw: String): EndpointValidationResult { @@ -61,7 +61,7 @@ internal class UnifiedPushEndpointValidator( } } -internal sealed interface EndpointValidationResult { +sealed interface EndpointValidationResult { data class Valid(val uri: URI, val addresses: List) : EndpointValidationResult data class Invalid(val reason: String) : EndpointValidationResult } diff --git a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt index ac0a1d74..048fa262 100644 --- a/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UnifiedPushSender.kt @@ -9,7 +9,7 @@ import okhttp3.RequestBody.Companion.toRequestBody import java.net.InetAddress import java.net.UnknownHostException -internal class UnifiedPushSender( +class UnifiedPushSender( private val endpointValidator: UnifiedPushEndpointValidator = UnifiedPushEndpointValidator(), client: OkHttpClient = OkHttpClient(), ) : PushEndpointSender { @@ -57,11 +57,11 @@ internal class UnifiedPushSender( } } -internal interface PushEndpointSender { +interface PushEndpointSender { suspend fun send(endpoint: String, eventId: String, payload: String): PushSendResult } -internal sealed interface PushSendResult { +sealed interface PushSendResult { data object Delivered : PushSendResult data object InvalidEndpoint : PushSendResult data class Retry(val statusCode: Int?) : PushSendResult diff --git a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt similarity index 90% rename from src/main/kotlin/dev/typetype/server/services/UrlValidator.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt index d7b488cc..1d3efb67 100644 --- a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt @@ -5,18 +5,18 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.net.InetAddress import java.net.URI -internal enum class ProxyProvider { +enum class ProxyProvider { YOUTUBE, BILIBILI, NICONICO, } -internal data class ProxyTarget( +data class ProxyTarget( val url: HttpUrl, val provider: ProxyProvider, ) -internal class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) +class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) private val BLOCKED_IPV4_RANGES = listOf( ipv4(0, 0, 0, 0) to 8, @@ -44,12 +44,12 @@ private val BLOCKED_IPV6_RANGES = listOf( byteArrayOf(0x20, 0x02) to 16, ) -internal fun validateProxyUrl(raw: String): String? = +fun validateProxyUrl(raw: String): String? = runCatching { requireProxyTarget(raw) } .exceptionOrNull() ?.message -internal fun requireProxyTarget(raw: String): ProxyTarget { +fun requireProxyTarget(raw: String): ProxyTarget { val uri = runCatching { URI(raw) }.getOrElse { throw ProxyTargetRejectedException("Malformed URL") } val scheme = uri.scheme?.lowercase() ?: throw ProxyTargetRejectedException("Missing URL scheme") if (scheme != "https") throw ProxyTargetRejectedException("Unsupported URL scheme: $scheme") @@ -65,7 +65,7 @@ internal fun requireProxyTarget(raw: String): ProxyTarget { return ProxyTarget(url, provider) } -internal fun providerForProxyHost(rawHost: String): ProxyProvider? { +fun providerForProxyHost(rawHost: String): ProxyProvider? { val host = rawHost.lowercase().trimEnd('.') return when { host.matchesHost("googlevideo.com") || @@ -81,7 +81,7 @@ internal fun providerForProxyHost(rawHost: String): ProxyProvider? { } } -internal fun isPublicProxyAddress(address: InetAddress): Boolean { +fun isPublicProxyAddress(address: InetAddress): Boolean { if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || address.isSiteLocalAddress || address.isMulticastAddress ) return false diff --git a/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/UserAdminService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UserAdminService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt new file mode 100644 index 00000000..975e6c9b --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -0,0 +1,11 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +fun VideoItem.isUpcomingAt(now: Long): Boolean = + !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now + +fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) + +fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = + isLive || isPostLive || isLiveContent || isUpcomingAt(now) diff --git a/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataItem.kt diff --git a/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/WatchLaterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt index 525c7e55..e741eb0d 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeChannelTabSort.kt @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.search.filter.FilterItem private val VALID_YOUTUBE_CHANNEL_TAB_SORTS = setOf("latest", "popular", "oldest") -internal fun String?.toYouTubeChannelTabSortFilter(): List? { +fun String?.toYouTubeChannelTabSortFilter(): List? { val normalized = this?.lowercase()?.takeIf { it.isNotBlank() } ?: return null if (normalized !in VALID_YOUTUBE_CHANNEL_TAB_SORTS) throw IllegalArgumentException("Invalid 'sort' parameter") return listOf(FilterItem(Filter.ITEM_IDENTIFIER_UNKNOWN, normalized)) diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt index 03297d24..b0a54d21 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleCache.kt @@ -7,7 +7,7 @@ import java.security.MessageDigest import java.time.Duration import java.util.HexFormat -internal class YouTubeSubtitleCache(private val sharedCache: CacheService?) { +class YouTubeSubtitleCache(private val sharedCache: CacheService?) { private val vod = BoundedExpiringCache( maxEntries = 128, maxWeight = MAX_MEMORY_BYTES, diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt index acb9e662..65b14152 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcher.kt @@ -12,11 +12,11 @@ import okhttp3.Response import java.io.IOException import kotlin.coroutines.resume -internal fun interface YouTubeSubtitleContentFetcher { +fun interface YouTubeSubtitleContentFetcher { suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult } -internal class OkHttpYouTubeSubtitleContentFetcher( +class OkHttpYouTubeSubtitleContentFetcher( private val client: OkHttpClient, ) : YouTubeSubtitleContentFetcher { override suspend fun fetch(url: String, format: YouTubeSubtitleFormat): YouTubeSubtitleFetchResult { @@ -33,7 +33,7 @@ internal class OkHttpYouTubeSubtitleContentFetcher( } } -internal class TokenYouTubeSubtitleContentFetcher( +class TokenYouTubeSubtitleContentFetcher( private val client: OkHttpClient, baseUrl: String, private val directFetcher: YouTubeSubtitleContentFetcher, @@ -92,9 +92,9 @@ private fun readSubtitleResponse(response: Response, format: YouTubeSubtitleForm } private const val YOUTUBE_ORIGIN = "https://m.youtube.com" -internal const val MAX_SUBTITLE_BYTES = 5 * 1024 * 1024 +const val MAX_SUBTITLE_BYTES = 5 * 1024 * 1024 -internal fun isValidSubtitlePayload(content: ByteArray, format: YouTubeSubtitleFormat): Boolean { +fun isValidSubtitlePayload(content: ByteArray, format: YouTubeSubtitleFormat): Boolean { if (content.isEmpty() || content.size > MAX_SUBTITLE_BYTES) return false val text = content.toString(Charsets.UTF_8).trimStart() return when (format) { diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt similarity index 85% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt index 1b60b8c3..3de2b424 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleContract.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.schabi.newpipe.extractor.MediaFormat -internal enum class YouTubeSubtitleFormat( +enum class YouTubeSubtitleFormat( val value: String, val mediaFormat: MediaFormat, val contentType: String, @@ -16,7 +16,7 @@ internal enum class YouTubeSubtitleFormat( } } -internal enum class YouTubeSubtitleVariant(val value: String) { +enum class YouTubeSubtitleVariant(val value: String) { Manual("manual"), Auto("auto"); @@ -25,7 +25,7 @@ internal enum class YouTubeSubtitleVariant(val value: String) { } } -internal data class YouTubeSubtitleSelection( +data class YouTubeSubtitleSelection( val videoId: String, val language: String, val variant: YouTubeSubtitleVariant, @@ -45,20 +45,20 @@ internal data class YouTubeSubtitleSelection( ).joinToString(":") } -internal data class ResolvedYouTubeSubtitle( +data class ResolvedYouTubeSubtitle( val content: String, val isUrl: Boolean, val isLive: Boolean, ) -internal sealed interface YouTubeSubtitleResolution { +sealed interface YouTubeSubtitleResolution { data class Ready(val track: ResolvedYouTubeSubtitle) : YouTubeSubtitleResolution data object NotFound : YouTubeSubtitleResolution data object Throttled : YouTubeSubtitleResolution data object Unavailable : YouTubeSubtitleResolution } -internal sealed interface YouTubeSubtitleFetchResult { +sealed interface YouTubeSubtitleFetchResult { data class Ready(val content: ByteArray) : YouTubeSubtitleFetchResult data object Expired : YouTubeSubtitleFetchResult data object Throttled : YouTubeSubtitleFetchResult @@ -66,7 +66,7 @@ internal sealed interface YouTubeSubtitleFetchResult { data object Unavailable : YouTubeSubtitleFetchResult } -internal sealed interface YouTubeSubtitleContentResult { +sealed interface YouTubeSubtitleContentResult { data class Ready( val content: ByteArray, val format: YouTubeSubtitleFormat, @@ -81,7 +81,7 @@ internal sealed interface YouTubeSubtitleContentResult { data object Unavailable : YouTubeSubtitleContentResult } -internal fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleSelection? { +fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleSelection? { val url = rawUrl.toHttpUrlOrNull()?.takeIf { it.isYouTubeTimedText() } ?: return null val videoId = url.queryParameter("v")?.trim().orEmpty() val sourceLanguage = url.queryParameter("lang")?.trim().orEmpty() @@ -100,13 +100,13 @@ internal fun subtitleSelectionFromTimedTextUrl(rawUrl: String): YouTubeSubtitleS ) } -internal fun isYouTubeTimedTextUrl(rawUrl: String): Boolean = +fun isYouTubeTimedTextUrl(rawUrl: String): Boolean = rawUrl.toHttpUrlOrNull()?.isYouTubeTimedText() == true -internal fun isValidYouTubeVideoId(value: String): Boolean = +fun isValidYouTubeVideoId(value: String): Boolean = value.length == 11 && value.all { it.isLetterOrDigit() || it == '_' || it == '-' } -internal fun isValidSubtitleTag(value: String): Boolean = +fun isValidSubtitleTag(value: String): Boolean = value.isNotEmpty() && value.length <= 64 && value.all { it.isLetterOrDigit() || it in "-_." } private fun okhttp3.HttpUrl.isYouTubeTimedText(): Boolean = diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt index 1e5ff2a2..2bef0a35 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.CompletableDeferred import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Semaphore -internal class YouTubeSubtitleDeliveryService( +class YouTubeSubtitleDeliveryService( private val resolver: YouTubeSubtitleTrackResolver, private val fetcher: YouTubeSubtitleContentFetcher, private val cache: YouTubeSubtitleCache, diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt index 0d1f5367..0a0f504c 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleService.kt @@ -13,7 +13,7 @@ import okhttp3.Response import java.io.IOException import kotlin.coroutines.resume -internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, private val baseUrl: String) { +class YouTubeSubtitleService(private val httpClient: OkHttpClient, private val baseUrl: String) { suspend fun fetchSubtitles(videoId: String): List = when (val result = fetchSubtitleInventory(videoId)) { @@ -55,7 +55,7 @@ internal class YouTubeSubtitleService(private val httpClient: OkHttpClient, priv } -internal sealed interface YouTubeSubtitleInventoryResult { +sealed interface YouTubeSubtitleInventoryResult { data class Ready(val tracks: List) : YouTubeSubtitleInventoryResult data object Unavailable : YouTubeSubtitleInventoryResult } diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt similarity index 66% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt index 90fda099..1955bc14 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClient.kt @@ -1,6 +1,6 @@ package dev.typetype.server.services -internal enum class YoutubePlayerClient(val value: String) { +enum class YoutubePlayerClient(val value: String) { MWEB("mweb"), VISIONOS("visionos"), TV_DOWNGRADED("tv_downgraded"), diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt similarity index 93% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt index 957af39c..a8d4e5a4 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamService.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -internal class YoutubePlayerClientFallbackStreamService( +class YoutubePlayerClientFallbackStreamService( private val delegate: StreamService, private val clients: List, ) : StreamService { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt similarity index 95% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt index 770951f2..b66046ca 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientScope.kt @@ -5,7 +5,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import org.schabi.newpipe.extractor.NewPipe -internal object YoutubePlayerClientScope { +object YoutubePlayerClientScope { private const val CONCURRENT_SABR_PERMITS = 64 private val permits = Semaphore(CONCURRENT_SABR_PERMITS, true) diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt similarity index 91% rename from src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt index ac916861..0b814010 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamService.kt @@ -3,7 +3,7 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -internal class YoutubePlayerClientStreamService( +class YoutubePlayerClientStreamService( private val delegate: StreamService, private val client: YoutubePlayerClient, ) : StreamService { diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserAudit.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserBridgeListener.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserClient.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserCompleteResult.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserConfig.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteBrowserSessionStore.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedChannelService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedCommentService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedPublicPlaylistService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSearchService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedSuggestionService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeScopedTrendingService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionHlsManifestService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt similarity index 87% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index 962039b7..7de1ffde 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -5,10 +5,10 @@ import dev.typetype.server.models.StreamResponse import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeout -internal const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" -internal const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" +const val YOUTUBE_SESSION_REQUIRED_CODE = "youtube_session_required" +const val YOUTUBE_SESSION_REQUIRED_ERROR = "Connect YouTube to access this video" -internal class YoutubeSessionSabrStreamService( +class YoutubeSessionSabrStreamService( private val metadataService: YoutubeSessionStreamService, private val infoService: AuthenticatedSabrInfoService, private val timeoutMs: Long = AuthenticatedSabrPolicy.STREAM_TIMEOUT_MS, @@ -38,7 +38,7 @@ internal class YoutubeSessionSabrStreamService( } } -internal fun ExtractionResult.requiresYoutubeSession(): Boolean = +fun ExtractionResult.requiresYoutubeSession(): Boolean = when (this) { is ExtractionResult.Success -> data.requiresMembership is ExtractionResult.BadRequest -> code == "age_restricted" || code == "members_only" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index d9ca60b6..a0759f72 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -79,4 +79,4 @@ class YoutubeSessionStreamService( } } -internal const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" +const val YOUTUBE_SESSION_RECONNECT_CODE = "youtube_session_needs_reconnect" diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCache.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCleanupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobEngine.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImportJobService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutImporterService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutParserService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewLookupService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutPreviewService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt rename to server-services/src/main/kotlin/dev/typetype/server/services/YoutubeTakeoutSignalImportService.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 96784ac5..b160148d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,6 +17,7 @@ include(":server-domain") include(":server-playback") include(":server-portability") include(":server-sabr") +include(":server-services") include(":server-token-gateway") val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") diff --git a/src/main/kotlin/dev/typetype/server/RequestObservability.kt b/src/main/kotlin/dev/typetype/server/RequestObservability.kt index a14c6db0..e7e2d3ca 100644 --- a/src/main/kotlin/dev/typetype/server/RequestObservability.kt +++ b/src/main/kotlin/dev/typetype/server/RequestObservability.kt @@ -10,15 +10,10 @@ import kotlinx.coroutines.asContextElement import kotlinx.coroutines.withContext import java.util.UUID -const val REQUEST_ID_HEADER = "X-Request-ID" - private val requestIdAttribute = AttributeKey("requestId") private val requestStartNanosAttribute = AttributeKey("requestStartNanos") -private val requestIdContext = ThreadLocal() private val requestIdRegex = Regex("^[A-Za-z0-9._-]{8,128}$") -fun currentRequestId(): String? = requestIdContext.get() - fun ApplicationCall.requestId(): String = attributeOrNull(requestIdAttribute) ?: currentRequestId() ?: "unknown" fun ApplicationCall.requestDurationMs(): Long { @@ -33,7 +28,7 @@ fun Application.installRequestObservability() { applicationCall.attributes.put(requestIdAttribute, requestId) applicationCall.attributes.put(requestStartNanosAttribute, System.nanoTime()) applicationCall.response.headers.append(REQUEST_ID_HEADER, requestId, safeOnly = false) - withContext(requestIdContext.asContextElement(requestId)) { + withContext(requestContextElement(requestId)) { try { proceed() } finally { diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt deleted file mode 100644 index e8f1837f..00000000 --- a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt +++ /dev/null @@ -1,11 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.VideoItem - -internal fun VideoItem.isUpcomingAt(now: Long): Boolean = - !isPostLive && RssVideoMetadata.publishedAtMillis(this) > now - -internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) - -internal fun VideoItem.isLiveContentOrUpcomingAt(now: Long): Boolean = - isLive || isPostLive || isLiveContent || isUpcomingAt(now) From d34e2421c58b4317b3bde347cdbc16b2cce6ae2b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 22:09:36 +0200 Subject: [PATCH 041/137] refactor: extract server-http routes module and slim app to bootstrap --- build.gradle.kts | 1 + server-http/build.gradle.kts | 31 +++++++++++++++++++ .../dev/typetype/server/ApplicationRoutes.kt | 2 +- .../server/ApplicationStreamRoutes.kt | 0 .../dev/typetype/server/RateLimitZones.kt | 11 +++++++ .../routes/AccessControlRouteHelpers.kt | 0 .../server/routes/AccountIdentityRoutes.kt | 0 .../server/routes/AccountProfilesRoutes.kt | 0 .../server/routes/AdminAllowListRoutes.kt | 0 .../routes/AdminAllowedPlaylistRoutes.kt | 0 .../dev/typetype/server/routes/AdminAuth.kt | 0 .../server/routes/AdminBugReportRoutes.kt | 0 .../server/routes/AdminManagedAccessRoutes.kt | 0 .../server/routes/AdminModeratorAuth.kt | 0 .../dev/typetype/server/routes/AdminRoutes.kt | 0 .../typetype/server/routes/AdminRssRoutes.kt | 0 .../server/routes/AdminSessionRoutes.kt | 0 .../routes/AdminUserAccessModeRoutes.kt | 0 .../server/routes/AdminUserAllowListRoutes.kt | 0 .../server/routes/AdminUserAllowedRoutes.kt | 0 .../server/routes/AdminUserSearchRoutes.kt | 0 .../server/routes/AllowedChannelsRoutes.kt | 0 .../server/routes/AudioOnlyByteRange.kt | 0 .../server/routes/AudioOnlyHeadResponse.kt | 0 .../routes/AudioOnlyProgressiveProbe.kt | 0 .../server/routes/AudioOnlyResponseMapper.kt | 0 .../typetype/server/routes/AudioOnlyRoutes.kt | 0 .../routes/AudioOnlySourceProxyResult.kt | 0 .../routes/AudioOnlySourceResolution.kt | 0 .../server/routes/AudioOnlyYoutubeVideoId.kt | 0 .../typetype/server/routes/AuthRouteModels.kt | 0 .../dev/typetype/server/routes/AuthRoutes.kt | 0 .../typetype/server/routes/AvatarRoutes.kt | 0 .../typetype/server/routes/BlockedRoutes.kt | 0 .../typetype/server/routes/BugReportRoutes.kt | 0 .../server/routes/BulletCommentRoutes.kt | 0 .../typetype/server/routes/ChannelRoutes.kt | 0 .../typetype/server/routes/CommentRoutes.kt | 0 .../server/routes/CustomAvatarRoutes.kt | 0 .../typetype/server/routes/DeArrowRoutes.kt | 0 .../routes/DownloaderGatewayArtifactProxy.kt | 0 .../server/routes/DownloaderGatewayHeaders.kt | 0 .../server/routes/DownloaderGatewayRoutes.kt | 0 .../routes/DownloaderGatewaySseProxy.kt | 0 .../routes/DownloaderGatewayStorageFailure.kt | 0 .../typetype/server/routes/FavoritesRoutes.kt | 0 .../typetype/server/routes/HistoryRoutes.kt | 0 .../routes/HomeRecommendationMetricsRoutes.kt | 0 .../server/routes/HomeRecommendationRoutes.kt | 0 .../routes/HomeRecommendationShortsRoutes.kt | 0 .../routes/InternalObservabilityRoutes.kt | 0 .../typetype/server/routes/ManifestRoutes.kt | 0 .../server/routes/NicoVideoProxyRoutes.kt | 0 .../typetype/server/routes/NoStoreResponse.kt | 0 .../server/routes/NotificationsRoutes.kt | 0 .../typetype/server/routes/OidcAuthRoutes.kt | 0 .../typetype/server/routes/PlaylistRoutes.kt | 0 .../typetype/server/routes/PodcastRoutes.kt | 0 .../server/routes/PortabilityJobRoutes.kt | 0 .../routes/PortabilityMultipartFailure.kt | 2 +- .../server/routes/PortabilityRouteSupport.kt | 2 +- .../server/routes/PortabilityRoutes.kt | 0 .../typetype/server/routes/PresenceRoutes.kt | 0 .../typetype/server/routes/ProfileRoutes.kt | 0 .../typetype/server/routes/ProgressRoutes.kt | 0 .../routes/ProviderMediaHandleRoutes.kt | 0 .../server/routes/ProxyResponseWriter.kt | 0 .../dev/typetype/server/routes/ProxyRoutes.kt | 0 .../routes/PublicHlsManifestTokenResolver.kt | 0 .../server/routes/PublicMetadataRoutes.kt | 0 .../server/routes/PublicPlaylistRoutes.kt | 0 .../server/routes/PushNotificationRoutes.kt | 0 .../routes/RecommendationEventsRoutes.kt | 0 .../routes/RecommendationFeedbackRoutes.kt | 0 .../routes/RecommendationOnboardingRoutes.kt | 0 .../typetype/server/routes/RegisterRoutes.kt | 0 .../typetype/server/routes/RestoreRoutes.kt | 0 .../server/routes/RouteUrlParameter.kt | 0 .../typetype/server/routes/RssFeedRoutes.kt | 0 .../server/routes/SabrContentTypes.kt | 0 .../server/routes/SabrDownloadRangeParser.kt | 0 .../server/routes/SabrFormatSelector.kt | 0 .../server/routes/SabrManifestAccess.kt | 0 .../routes/SabrManifestAccessResolver.kt | 0 .../server/routes/SabrManifestHandler.kt | 0 .../server/routes/SabrManifestResponder.kt | 0 .../server/routes/SabrManifestResponse.kt | 0 .../server/routes/SabrMediaResponseWriter.kt | 0 .../routes/SabrPlaybackAccessValidator.kt | 0 .../server/routes/SabrPlaybackClientState.kt | 0 .../server/routes/SabrPlaybackHandler.kt | 0 .../routes/SabrPlaybackManifestResponse.kt | 0 .../server/routes/SabrPlaybackModels.kt | 0 .../server/routes/SabrPlaybackPaths.kt | 0 .../server/routes/SabrPlaybackRate.kt | 0 .../server/routes/SabrPlaybackRecovery.kt | 0 .../routes/SabrPlaybackResponseFactory.kt | 0 .../server/routes/SabrPlaybackStateHandler.kt | 0 .../routes/SabrPlaybackWindowBuilder.kt | 0 .../routes/SabrPlaybackWindowHandler.kt | 0 .../routes/SabrPlaybackWindowSegmentTiming.kt | 0 .../server/routes/SabrPlaybackWindowTiming.kt | 0 .../routes/SabrProgressivePlaybackWindow.kt | 0 .../dev/typetype/server/routes/SabrRoutes.kt | 0 .../server/routes/SabrSegmentHandler.kt | 0 .../routes/SabrSessionDescriptorHandler.kt | 0 .../server/routes/SabrSessionStateHandler.kt | 0 .../server/routes/SabrStreamContractFilter.kt | 0 .../server/routes/SavedPlaylistRoutes.kt | 0 .../server/routes/SearchHistoryRoutes.kt | 0 .../typetype/server/routes/SearchRoutes.kt | 0 .../server/routes/SessionActivityRoutes.kt | 0 .../typetype/server/routes/SettingsRoutes.kt | 0 .../server/routes/StoryboardProxyRoutes.kt | 0 .../server/routes/StreamDeliveryMode.kt | 0 .../server/routes/StreamRouteDependencies.kt | 0 .../typetype/server/routes/StreamRoutes.kt | 0 .../server/routes/SubscriptionFeedRoutes.kt | 2 +- .../SubscriptionGroupMembershipRequestBody.kt | 0 .../server/routes/SubscriptionGroupsRoutes.kt | 0 .../routes/SubscriptionSelectionParameter.kt | 0 .../routes/SubscriptionShortsFeedRoutes.kt | 0 .../server/routes/SubscriptionsRoutes.kt | 0 .../server/routes/SuggestionRoutes.kt | 0 .../server/routes/TooManyRequestsBody.kt | 10 ++++++ .../typetype/server/routes/TrendingRoutes.kt | 0 .../server/routes/TypeTypeBackupRoutes.kt | 0 .../dev/typetype/server/routes/UserAuth.kt | 0 .../typetype/server/routes/UserDataRoutes.kt | 0 .../server/routes/WatchLaterRoutes.kt | 0 .../server/routes/YouTubeSubtitleResponse.kt | 2 +- .../server/routes/YouTubeSubtitleRoutes.kt | 0 .../routes/YoutubeRemoteBrowserRoutes.kt | 0 .../server/routes/YoutubeSessionRoutes.kt | 0 .../routes/YoutubeTakeoutImportRoutes.kt | 0 server-services/build.gradle.kts | 2 ++ .../kotlin/dev/typetype/server/AppMetrics.kt | 0 .../server/ExtractionServiceRegistry.kt | 0 .../server/HomeRecommendationServices.kt | 0 .../typetype/server/RequestObservability.kt | 0 .../dev/typetype/server/ServiceRegistry.kt | 2 +- settings.gradle.kts | 1 + .../kotlin/dev/typetype/server/Plugins.kt | 15 ++------- 143 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 server-http/build.gradle.kts rename {src => server-http/src}/main/kotlin/dev/typetype/server/ApplicationRoutes.kt (99%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt (100%) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminAuth.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt (77%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt (96%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt (98%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt (100%) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/UserAuth.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt (97%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt (100%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/AppMetrics.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/RequestObservability.kt (100%) rename {src => server-services/src}/main/kotlin/dev/typetype/server/ServiceRegistry.kt (99%) diff --git a/build.gradle.kts b/build.gradle.kts index 22e1d84a..fe8e3841 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -34,6 +34,7 @@ dependencies { implementation(project(":server-portability")) implementation(project(":server-playback")) implementation(project(":server-services")) + implementation(project(":server-http")) implementation(project(":server-sabr")) implementation(project(":server-token-gateway")) implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.2")) diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts new file mode 100644 index 00000000..c10602ad --- /dev/null +++ b/server-http/build.gradle.kts @@ -0,0 +1,31 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-services")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-downloader")) + implementation(project(":server-playback")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("io.ktor:ktor-server-rate-limit-jvm:3.5.2") + implementation("io.ktor:ktor-utils-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("com.squareup.okhttp3:okhttp:5.5.0") +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index a8d024cd..2e9422e3 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -53,7 +53,7 @@ import io.ktor.server.application.Application import io.ktor.server.plugins.ratelimit.rateLimit import io.ktor.server.routing.routing -internal fun Application.installApplicationRoutes( +fun Application.installApplicationRoutes( svc: ServiceRegistry, authService: AuthService, authSessionConfig: AuthSessionConfig, diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt b/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt new file mode 100644 index 00000000..173179ad --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/RateLimitZones.kt @@ -0,0 +1,11 @@ +package dev.typetype.server + +import io.ktor.server.plugins.ratelimit.RateLimitName + +val EXTRACTION_ZONE = RateLimitName("extraction") +val DEARROW_ZONE = RateLimitName("dearrow") +val STREAMS_ZONE = RateLimitName("streams") +val CHANNEL_ZONE = RateLimitName("channel") +val PROXY_ZONE = RateLimitName("proxy") +val PROXY_STORYBOARD_ZONE = RateLimitName("proxy-storyboard") +val USER_DATA_ZONE = RateLimitName("user-data") diff --git a/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccessControlRouteHelpers.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccountIdentityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AccountProfilesRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AllowedChannelsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyByteRange.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyHeadResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyProgressiveProbe.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyResponseMapper.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceProxyResult.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlySourceResolution.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AudioOnlyYoutubeVideoId.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AuthRouteModels.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AuthRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/AvatarRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BlockedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BugReportRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/BulletCommentRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/CommentRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/CustomAvatarRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayArtifactProxy.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayHeaders.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewaySseProxy.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/DownloaderGatewayStorageFailure.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/FavoritesRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HistoryRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationMetricsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/HomeRecommendationShortsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ManifestRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/NicoVideoProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/NotificationsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/OidcAuthRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PodcastRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityJobRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt similarity index 77% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt index a9751289..98a6a4a5 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt @@ -2,7 +2,7 @@ package dev.typetype.server.routes import java.io.IOException -internal fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && ( +fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && ( message?.let { (it.startsWith("Multipart content length exceeds limit ") && "formFieldLimit" in it) || (it.startsWith("Limit of ") && " bytes exceeded while searching for " in it) diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt similarity index 96% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt index 11c9bd47..73cf0fc6 100644 --- a/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt @@ -16,7 +16,7 @@ internal fun parsePortabilityFormat(value: String?): PortabilityFormat? { ?: throw IllegalArgumentException("Unsupported portability format") } -internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) { +suspend fun ApplicationCall.respondPortabilityError(error: Exception) { if (error is kotlinx.coroutines.CancellationException) throw error if (error.isMultipartSizeLimit()) { respondPortabilityError(PortabilityUploadTooLargeException()) diff --git a/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PortabilityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PresenceRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProfileRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProviderMediaHandleRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProxyResponseWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/ProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicHlsManifestTokenResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicMetadataRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationEventsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationFeedbackRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RecommendationOnboardingRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RegisterRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RestoreRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrContentTypes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrDownloadRangeParser.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccess.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestAccessResolver.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponder.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrMediaResponseWriter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidator.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackClientState.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackManifestResponse.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPaths.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRate.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRecovery.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackResponseFactory.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindow.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSegmentHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionStateHandler.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SavedPlaylistRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SearchHistoryRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SearchRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SessionActivityRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SettingsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StoryboardProxyRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt similarity index 98% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index abdfb136..e4cd3338 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -2,7 +2,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.SubscriptionFeedPreparingResponse -import dev.typetype.server.preserveTooManyRequestsBody +import dev.typetype.server.routes.preserveTooManyRequestsBody import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupMembershipRequestBody.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionShortsFeedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/SuggestionRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt new file mode 100644 index 00000000..bddccde8 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/TooManyRequestsBody.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.routes + +import io.ktor.server.application.ApplicationCall +import io.ktor.util.AttributeKey + +val TooManyRequestsBodyAttribute = AttributeKey("preserveTooManyRequestsBody") + +fun ApplicationCall.preserveTooManyRequestsBody() { + attributes.put(TooManyRequestsBodyAttribute, Unit) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/TrendingRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/TypeTypeBackupRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/UserAuth.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/UserAuth.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/WatchLaterRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt similarity index 97% rename from src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt index ddd76ea1..60201afb 100644 --- a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleResponse.kt @@ -1,7 +1,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse -import dev.typetype.server.preserveTooManyRequestsBody +import dev.typetype.server.routes.preserveTooManyRequestsBody import dev.typetype.server.services.YouTubeSubtitleContentResult import io.ktor.http.ContentType import io.ktor.http.HttpHeaders diff --git a/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YouTubeSubtitleRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeRemoteBrowserRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeSessionRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt rename to server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeTakeoutImportRoutes.kt diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts index fedf7105..6e2d8339 100644 --- a/server-services/build.gradle.kts +++ b/server-services/build.gradle.kts @@ -8,6 +8,8 @@ dependencies { api(project(":server-core")) api(project(":server-domain")) api(project(":server-playback")) + api(project(":server-auth")) + api(project(":server-downloader")) implementation(project(":server-auth")) implementation(project(":server-cache")) implementation(project(":server-db")) diff --git a/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/AppMetrics.kt rename to server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt diff --git a/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt b/server-services/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt rename to server-services/src/main/kotlin/dev/typetype/server/HomeRecommendationServices.kt diff --git a/src/main/kotlin/dev/typetype/server/RequestObservability.kt b/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/RequestObservability.kt rename to server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt similarity index 99% rename from src/main/kotlin/dev/typetype/server/ServiceRegistry.kt rename to server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 1ca8c71b..f4bc5200 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -41,7 +41,7 @@ import dev.typetype.server.services.VideoMetadataResolver import dev.typetype.server.services.WatchLaterService import dev.typetype.server.services.YoutubeTakeoutFactory import java.net.ProxySelector -internal class ServiceRegistry( +class ServiceRegistry( cache: DragonflyService, subtitleServiceUrl: String, youtubeSessionEncryptionKey: String?, diff --git a/settings.gradle.kts b/settings.gradle.kts index b160148d..fa0c0ddd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,6 +14,7 @@ include(":server-core") include(":server-db") include(":server-downloader") include(":server-domain") +include(":server-http") include(":server-playback") include(":server-portability") include(":server-sabr") diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index ef71d1cf..85f61617 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -21,6 +21,7 @@ import io.ktor.server.request.path import io.ktor.server.response.respond import io.ktor.server.websocket.WebSockets import io.ktor.util.AttributeKey +import dev.typetype.server.routes.TooManyRequestsBodyAttribute import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory import kotlin.time.Duration.Companion.minutes @@ -34,15 +35,7 @@ private const val PROXY_STORYBOARD_RATE_LIMIT = 1200 private const val USER_DATA_RATE_LIMIT = 120 private const val MAX_WEBSOCKET_FRAME_BYTES = 64L * 1024L * 1024L private val RATE_LIMIT_WINDOW = 1.minutes -private val preserveTooManyRequestsBodyAttribute = AttributeKey("preserveTooManyRequestsBody") -val EXTRACTION_ZONE = RateLimitName("extraction") -val DEARROW_ZONE = RateLimitName("dearrow") -val STREAMS_ZONE = RateLimitName("streams") -val CHANNEL_ZONE = RateLimitName("channel") -val PROXY_ZONE = RateLimitName("proxy") -val PROXY_STORYBOARD_ZONE = RateLimitName("proxy-storyboard") -val USER_DATA_ZONE = RateLimitName("user-data") fun Application.configurePlugins(authService: AuthService) { installRequestObservability() @@ -101,15 +94,13 @@ fun Application.configurePlugins(authService: AuthService) { configureStatusPages() } -internal fun ApplicationCall.preserveTooManyRequestsBody() { - attributes.put(preserveTooManyRequestsBodyAttribute, Unit) -} + internal fun Application.configureStatusPages() { val log = LoggerFactory.getLogger("RequestLogger") install(StatusPages) { status(HttpStatusCode.TooManyRequests) { call, status -> - if (call.attributes.contains(preserveTooManyRequestsBodyAttribute)) return@status + if (call.attributes.contains(TooManyRequestsBodyAttribute)) return@status if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") call.respond(status, ErrorResponse("Too many requests", "rate_limited")) } From 4c6193f5cf1c09830fe8c5369f01557f7ca91ede Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 22:15:47 +0200 Subject: [PATCH 042/137] refactor: extract server-admin into Gradle module --- build.gradle.kts | 1 + server-admin/build.gradle.kts | 29 +++++++++++++++++++ .../server/routes/AdminAllowListRoutes.kt | 0 .../routes/AdminAllowedPlaylistRoutes.kt | 0 .../dev/typetype/server/routes/AdminAuth.kt | 0 .../server/routes/AdminBugReportRoutes.kt | 0 .../server/routes/AdminManagedAccessRoutes.kt | 0 .../server/routes/AdminModeratorAuth.kt | 0 .../dev/typetype/server/routes/AdminRoutes.kt | 0 .../typetype/server/routes/AdminRssRoutes.kt | 0 .../server/routes/AdminSessionRoutes.kt | 0 .../routes/AdminUserAccessModeRoutes.kt | 0 .../server/routes/AdminUserAllowListRoutes.kt | 0 .../server/routes/AdminUserAllowedRoutes.kt | 0 .../server/routes/AdminUserSearchRoutes.kt | 0 .../routes/InternalObservabilityRoutes.kt | 0 .../typetype/server/routes/NoStoreResponse.kt | 2 +- .../server/routes/RouteUrlParameter.kt | 2 +- .../server/routes/RssErrorResponse.kt | 17 +++++++++++ server-http/build.gradle.kts | 1 + .../typetype/server/routes/RssFeedRoutes.kt | 9 ------ settings.gradle.kts | 1 + 22 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 server-admin/build.gradle.kts rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt (100%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt (80%) rename {server-http => server-admin}/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt (91%) create mode 100644 server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt diff --git a/build.gradle.kts b/build.gradle.kts index fe8e3841..becc6cba 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -25,6 +25,7 @@ repositories { } dependencies { + implementation(project(":server-admin")) implementation(project(":server-auth")) implementation(project(":server-cache")) implementation(project(":server-core")) diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts new file mode 100644 index 00000000..1ba3a81d --- /dev/null +++ b/server-admin/build.gradle.kts @@ -0,0 +1,29 @@ +plugins { + kotlin("jvm") + kotlin("plugin.serialization") +} + +dependencies { + api(project(":server-core")) + api(project(":server-services")) + implementation(project(":server-auth")) + implementation(project(":server-cache")) + implementation(project(":server-db")) + implementation(project(":server-domain")) + implementation(project(":server-downloader")) + implementation(project(":server-playback")) + implementation(project(":server-portability")) + implementation(project(":server-sabr")) + implementation(project(":server-token-gateway")) + implementation("io.ktor:ktor-server-core-jvm:3.5.2") + implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") + implementation("org.slf4j:slf4j-api:2.0.16") +} + +kotlin { + jvmToolchain(25) +} diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowListRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAllowedPlaylistRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminAuth.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminBugReportRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminManagedAccessRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminModeratorAuth.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminSessionRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAccessModeRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowListRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserAllowedRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/AdminUserSearchRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt similarity index 100% rename from server-http/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/InternalObservabilityRoutes.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt similarity index 80% rename from server-http/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt index 5fdc23b5..2c26ac21 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/NoStoreResponse.kt @@ -4,7 +4,7 @@ import io.ktor.http.HttpHeaders import io.ktor.server.application.ApplicationCall import io.ktor.server.response.respond -internal suspend fun ApplicationCall.respondNoStore(value: Any): Unit { +suspend fun ApplicationCall.respondNoStore(value: Any): Unit { response.headers.append(HttpHeaders.CacheControl, "no-store, no-cache, must-revalidate, max-age=0", safeOnly = false) respond(value) } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt similarity index 91% rename from server-http/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt rename to server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt index 92c4dee1..dcbf95f2 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/RouteUrlParameter.kt @@ -3,7 +3,7 @@ package dev.typetype.server.routes import io.ktor.server.application.ApplicationCall import io.ktor.server.request.queryString -internal fun ApplicationCall.urlTailParameter(name: String): String? { +fun ApplicationCall.urlTailParameter(name: String): String? { val pathUrl = parameters.getAll(name)?.joinToString("/") ?: return null val normalizedPathUrl = pathUrl.withUrlSchemeSlashes() val queryString = request.queryString() diff --git a/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt b/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt new file mode 100644 index 00000000..30d3052e --- /dev/null +++ b/server-admin/src/main/kotlin/dev/typetype/server/routes/RssErrorResponse.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.RssFeedException +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond + +suspend fun ApplicationCall.respondRssError(error: RssFeedException) { + val status = when (error.code) { + "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound + "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden + "rss_feed_limit_reached" -> HttpStatusCode.Conflict + else -> HttpStatusCode.BadRequest + } + respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) +} diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts index c10602ad..a23eee47 100644 --- a/server-http/build.gradle.kts +++ b/server-http/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { api(project(":server-core")) api(project(":server-services")) + api(project(":server-admin")) implementation(project(":server-auth")) implementation(project(":server-cache")) implementation(project(":server-db")) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt index b36b0faa..3fe90e12 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt @@ -82,15 +82,6 @@ fun Route.rssPublicRoutes(service: RssFeedReaderService) { } } -internal suspend fun ApplicationCall.respondRssError(error: RssFeedException) { - val status = when (error.code) { - "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound - "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden - "rss_feed_limit_reached" -> HttpStatusCode.Conflict - else -> HttpStatusCode.BadRequest - } - respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) -} private suspend inline fun ApplicationCall.withRssUser( authService: AuthService, diff --git a/settings.gradle.kts b/settings.gradle.kts index fa0c0ddd..7caea1eb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -8,6 +8,7 @@ dependencyResolutionManagement { } } +include(":server-admin") include(":server-auth") include(":server-cache") include(":server-core") From ad8d0d8285274d69074b0d3320aa65ba640b909b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 18 Sep 2026 22:54:25 +0200 Subject: [PATCH 043/137] refactor: co-locate tests with owning modules and add test-fixtures --- build.gradle.kts | 19 ++----- server-admin/build.gradle.kts | 16 ++++++ .../server/AdminManagedAccessRoutesTest.kt | 0 .../dev/typetype/server/AdminRssRoutesTest.kt | 0 .../server/AdminUserAccessModeRoutesTest.kt | 0 .../server/InternalObservabilityRoutesTest.kt | 0 server-auth/build.gradle.kts | 5 ++ .../server/AccountIdentityServiceTest.kt | 0 .../typetype/server/AuthCookieHelpersTest.kt | 0 .../dev/typetype/server/AvatarServiceTest.kt | 0 .../typetype/server/OidcAuthServiceTest.kt | 0 server-cache/build.gradle.kts | 10 ++++ .../dev/typetype/server/FakeCacheService.kt | 0 server-core/build.gradle.kts | 10 ++++ .../server/services/AdminSettingsCache.kt | 18 +++++++ .../dev/typetype/server/TestFixtures.kt | 0 .../server/TestStreamResponseFixture.kt | 0 .../typetype/server/TestVideoItemFixture.kt | 0 server-db/build.gradle.kts | 15 ++++++ ...DatabaseCollectionMetadataMigrationTest.kt | 0 ...atabaseImportedMediaRepairMigrationTest.kt | 0 .../dev/typetype/server/TestDatabase.kt | 6 +-- server-domain/build.gradle.kts | 8 +++ server-http/build.gradle.kts | 20 +++++++ .../dev/typetype/server/CompressionConfig.kt | 0 .../dev/typetype/server/StatusPagesConfig.kt | 43 +++++++++++++++ .../server/AccountProfilesRoutesTest.kt | 0 .../server/AllowedChannelsRoutesTest.kt | 0 .../typetype/server/AudioOnlyByteRangeTest.kt | 0 .../AudioOnlyPlayableContractRoutesTest.kt | 0 .../AudioOnlyProgressiveContractRoutesTest.kt | 0 .../AudioOnlySourceOpenRangeRoutesTest.kt | 0 .../server/AudioOnlySourceRoutesTest.kt | 0 .../dev/typetype/server/BlockedRoutesTest.kt | 0 .../typetype/server/BugReportRoutesTest.kt | 0 .../server/BulletCommentRoutesTest.kt | 0 .../dev/typetype/server/ChannelRoutesTest.kt | 0 .../dev/typetype/server/CommentRoutesTest.kt | 0 .../server/DeArrowThumbnailContentTypeTest.kt | 0 .../DownloaderGatewayArtifactProxyTest.kt | 0 .../server/DownloaderGatewayRoutesTest.kt | 0 .../server/FakeRemoteBrowserClient.kt | 0 .../typetype/server/FavoritesRoutesTest.kt | 0 .../dev/typetype/server/HistoryRoutesTest.kt | 0 .../server/HomeRecommendationRoutesTest.kt | 0 .../HomeRecommendationShortsRoutesTest.kt | 0 .../dev/typetype/server/ManifestRoutesTest.kt | 0 .../server/NicoVideoProxyRoutesTest.kt | 0 .../server/NotificationsRoutesTest.kt | 0 .../dev/typetype/server/OidcAuthRoutesTest.kt | 0 .../dev/typetype/server/PlaylistRoutesTest.kt | 0 .../dev/typetype/server/PodcastRoutesTest.kt | 0 .../PortabilityArchiveReproductionTest.kt | 0 .../server/PortabilityMultipartFailureTest.kt | 0 .../typetype/server/PortabilityRoutesTest.kt | 0 .../dev/typetype/server/PresenceRoutesTest.kt | 0 .../dev/typetype/server/ProfileRoutesTest.kt | 0 .../dev/typetype/server/ProgressRoutesTest.kt | 0 .../server/ProviderMediaHandleRoutesTest.kt | 0 .../dev/typetype/server/ProxyRoutesTest.kt | 0 .../server/PublicPlaylistRoutesTest.kt | 0 .../dev/typetype/server/RestoreRoutesTest.kt | 0 .../SabrHttpRandomAccessInitProbeTest.kt | 0 .../server/SabrMediaResponseWriterTest.kt | 0 .../typetype/server/SabrRoutesAccessTest.kt | 0 .../server/SabrStreamContractFilterTest.kt | 0 .../server/SavedPlaylistRoutesTest.kt | 0 .../server/SearchHistoryRoutesTest.kt | 0 .../dev/typetype/server/SearchRoutesTest.kt | 0 .../dev/typetype/server/SettingsRoutesTest.kt | 0 .../server/StoryboardProxyRoutesTest.kt | 0 .../dev/typetype/server/StreamRoutesTest.kt | 0 .../server/SubscriptionFeedRoutesTest.kt | 0 .../server/SubscriptionGroupsRoutesTest.kt | 0 .../SubscriptionShortsFeedRoutesTest.kt | 0 ...SubscriptionsCanonicalizationRoutesTest.kt | 0 .../SubscriptionsDeleteProxyRoutesTest.kt | 0 .../server/SubscriptionsRoutesTest.kt | 0 .../typetype/server/SuggestionRoutesTest.kt | 0 .../dev/typetype/server/TrendingRoutesTest.kt | 0 .../typetype/server/WatchLaterRoutesTest.kt | 0 .../server/YouTubeSubtitleProxyRoutesTest.kt | 0 .../YoutubeRemoteBrowserCompleteRoutesTest.kt | 0 .../server/YoutubeRemoteBrowserRoutesTest.kt | 0 .../server/YoutubeSessionRoutesTest.kt | 0 .../server/YoutubeTakeoutImportRoutesTest.kt | 0 .../server/routes/SabrLiveGapWindowTest.kt | 0 .../SabrLivePlaybackContinuityProbeTest.kt | 0 .../routes/SabrLivePlaybackOverlapTest.kt | 0 .../SabrLivePlaybackWindowBuilderTest.kt | 0 .../SabrLiveRoundedBoundaryWindowTest.kt | 0 .../routes/SabrPlaybackAccessValidatorTest.kt | 0 .../routes/SabrPlaybackAudioOnlyWindowTest.kt | 0 .../server/routes/SabrPlaybackRateTest.kt | 0 .../server/routes/SabrPlaybackRecoveryTest.kt | 0 .../routes/SabrPlaybackSeekRouteTest.kt | 0 .../routes/SabrPlaybackWindowBuilderTest.kt | 0 .../SabrProgressivePlaybackWindowTest.kt | 0 .../routes/SabrStreamContractFilterTest.kt | 0 .../SabrPlaybackGranularRoutesTest.kt | 0 .../SabrPlaybackLiveGapServiceTest.kt | 0 server-playback/build.gradle.kts | 4 ++ .../server/YoutubeSessionTokenScopeTest.kt | 0 .../services/TestStreamResponseFixture.kt | 53 ------------------- server-portability/build.gradle.kts | 12 +++++ ...YoutubeTakeoutActivitySignalServiceTest.kt | 0 .../YoutubeTakeoutImportProgressTest.kt | 0 .../server/YoutubeTakeoutIssueServiceTest.kt | 0 .../server/YoutubeTakeoutPathHintsTest.kt | 0 .../portability/FlowPortabilityAdapterTest.kt | 0 .../GrayjayPortabilityAdapterTest.kt | 0 .../InvidiousPortabilityAdapterTest.kt | 0 .../LibreTubePortabilityAdapterTest.kt | 0 .../MaterialiousPortabilityAdapterTest.kt | 0 .../NewPipePortabilityAdapterTest.kt | 0 .../portability/OpmlPortabilityAdapterTest.kt | 0 .../PipedPortabilityAdapterTest.kt | 0 .../portability/PortabilityEngineTest.kt | 0 .../portability/PortabilityJobStoreTest.kt | 0 .../portability/PortabilityProgressTest.kt | 0 .../portability/PortabilitySpoolTest.kt | 0 .../TypeTypePortabilityAdapterTest.kt | 0 .../TypeTypePortabilityHistoryImportTest.kt | 0 .../ViewTubePortabilityAdapterTest.kt | 0 .../YoutubeTakeoutCsvSchemaDetectorTest.kt | 0 ...YoutubeTakeoutHtmlPortabilityReaderTest.kt | 0 .../YoutubeTakeoutPortabilityAdapterTest.kt | 0 server-sabr/build.gradle.kts | 2 +- server-services/build.gradle.kts | 26 +++++++++ .../server/services/AdminSettingsService.kt | 11 ++-- .../typetype/server/BilibiliRangeProxyTest.kt | 0 .../server/BilibiliRelatedServiceTest.kt | 0 .../server/CachedStreamServiceTest.kt | 0 .../server/CachedSuggestionServiceTest.kt | 0 .../typetype/server/ChannelTabResolverTest.kt | 0 .../server/CustomAvatarServiceTest.kt | 0 .../dev/typetype/server/DeArrowServiceTest.kt | 0 .../server/ExtractionErrorSanitizerTest.kt | 0 .../server/ExtractionResilienceTest.kt | 0 .../typetype/server/GitHubIssueServiceTest.kt | 0 .../server/HistoryProgressMapperTest.kt | 0 .../HomeRecommendationCandidateServiceTest.kt | 0 .../HomeRecommendationCursorCodecTest.kt | 0 ...HomeRecommendationLiveTitleDetectorTest.kt | 0 .../server/HomeRecommendationMixerTest.kt | 0 .../HomeRecommendationPoolBuilderTest.kt | 0 .../HomeRecommendationPoolRotationTest.kt | 0 ...omeRecommendationShortsDeduplicatorTest.kt | 0 .../HomeRecommendationShortsFallbackTest.kt | 0 .../HomeRecommendationShortsRefresherTest.kt | 0 .../HomeRecommendationTemporalBoostTest.kt | 0 .../typetype/server/LiveStreamTypesTest.kt | 0 .../server/ManifestRepresentationIdsTest.kt | 0 .../typetype/server/ManifestServiceTest.kt | 0 .../server/NativeManifestUtilsTest.kt | 0 .../server/NicoVideoProxyServiceTest.kt | 0 .../server/PipePipeBackupSqliteReaderTest.kt | 0 .../PipePipeBackupTimelineNormalizerTest.kt | 0 .../typetype/server/PlaylistServiceTest.kt | 0 .../server/ProviderMediaHandleServiceTest.kt | 0 .../typetype/server/ProxyHttpExecutorTest.kt | 0 .../typetype/server/PublicCachePolicyTest.kt | 0 .../typetype/server/PublishedAtMapperTest.kt | 0 .../typetype/server/PushDeviceRegistryTest.kt | 0 .../PushNotificationDeliveryStoreTest.kt | 0 .../server/PushNotificationSupportTest.kt | 0 .../server/RequestObservabilityTest.kt | 0 .../server/RetryingProxyInputStreamTest.kt | 0 .../server/RssDocumentRendererTest.kt | 0 .../server/RssFeedManagementServiceTest.kt | 0 .../dev/typetype/server/RssFeedSecretTest.kt | 0 .../typetype/server/RssVideoTypeFilterTest.kt | 0 .../typetype/server/SearchContentKindTest.kt | 0 .../server/SearchFilterMappersTest.kt | 0 .../SignedHlsManifestTokenServiceTest.kt | 0 .../server/StoryboardProxyDetectorTest.kt | 0 .../server/StreamAudioContractResolverTest.kt | 0 .../server/StreamCacheTtlResolverTest.kt | 0 .../server/StreamExtractionErrorMapperTest.kt | 0 .../SubscriptionFeedAvailabilityTest.kt | 0 .../server/SubscriptionFeedCacheKeysTest.kt | 0 .../server/SubscriptionFeedOrdererTest.kt | 0 .../SubscriptionFeedSelectionStoreTest.kt | 0 .../server/SubscriptionFeedServiceTest.kt | 0 .../SubscriptionFeedSnapshotStoreTest.kt | 0 .../server/SubscriptionGroupsServiceTest.kt | 0 .../server/TypeTypeBackupServiceTest.kt | 0 .../UnifiedPushEndpointValidatorTest.kt | 0 .../dev/typetype/server/UrlValidatorTest.kt | 0 .../YoutubeRemoteLoginReadinessServiceTest.kt | 0 .../YoutubeSessionHlsManifestServiceTest.kt | 0 .../server/YoutubeSessionStreamServiceTest.kt | 0 .../server/YoutubeTakeoutParserServiceTest.kt | 0 .../services/BlockedContentProfileTest.kt | 0 .../server/services/HomeWarmupTrackerTest.kt | 0 .../services/OpenMojiProxyServiceTest.kt | 0 .../server/services/RelatedItemMappersTest.kt | 0 .../SabrAdaptiveInitializationTest.kt | 0 .../TypetypeTokenSabrTokenClientTest.kt | 0 ...petypeYoutubeSessionPoTokenProviderTest.kt | 0 .../YouTubeSubtitleContentFetcherTest.kt | 0 .../services/YouTubeSubtitleContractTest.kt | 0 .../YouTubeSubtitleDeliveryServiceTest.kt | 0 .../services/YouTubeSubtitleServiceTest.kt | 0 ...bePlayerClientFallbackStreamServiceTest.kt | 0 .../YoutubePlayerClientStreamServiceTest.kt | 0 .../services/YoutubeTakeoutImportCacheTest.kt | 0 .../server/ActiveSessionTestFixtures.kt | 0 .../dev/typetype/server/FakeChannelService.kt | 0 .../server/GitHubIssueServiceTestReports.kt | 2 +- .../server/HomeRecommendationItemFixtures.kt | 2 +- .../server/HomeRecommendationTestFixtures.kt | 0 .../server/PipePipeBackupTestFixtures.kt | 0 .../server/SubscriptionFeedTestFixtures.kt | 2 +- .../typetype/server/TestCommentFixtures.kt | 0 server-test-support/build.gradle.kts | 26 +++++++++ .../dev/typetype/server}/SabrProbeConfig.kt | 2 + .../typetype/server}/SabrProbeDiagnostics.kt | 14 ++--- .../dev/typetype/server}/SabrProbeFetch.kt | 8 +-- .../typetype/server}/SabrProbeFetchResult.kt | 9 +++- server-token-gateway/build.gradle.kts | 2 + .../YoutubeRemoteBrowserMessageGuardTest.kt | 0 .../YoutubeSessionCookieNormalizerTest.kt | 0 .../YoutubeSessionCredentialValidatorTest.kt | 0 .../server/YoutubeSessionCryptoTest.kt | 0 settings.gradle.kts | 1 + .../kotlin/dev/typetype/server/Plugins.kt | 26 +-------- .../typetype/server/services/SabrProbeTest.kt | 7 --- 228 files changed, 254 insertions(+), 125 deletions(-) rename {src => server-admin/src}/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt (100%) rename {src => server-admin/src}/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt (100%) rename {src => server-admin/src}/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt (100%) rename {src => server-admin/src}/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt (100%) rename {src => server-auth/src}/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt (100%) rename {src => server-auth/src}/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt (100%) rename {src => server-auth/src}/test/kotlin/dev/typetype/server/AvatarServiceTest.kt (100%) rename {src => server-auth/src}/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt (100%) rename {src/test => server-cache/src/testFixtures}/kotlin/dev/typetype/server/FakeCacheService.kt (100%) create mode 100644 server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt rename {src/test => server-core/src/testFixtures}/kotlin/dev/typetype/server/TestFixtures.kt (100%) rename {src/test => server-core/src/testFixtures}/kotlin/dev/typetype/server/TestStreamResponseFixture.kt (100%) rename {src/test => server-core/src/testFixtures}/kotlin/dev/typetype/server/TestVideoItemFixture.kt (100%) rename {src => server-db/src}/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt (100%) rename {src => server-db/src}/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt (100%) rename {src/test => server-db/src/testFixtures}/kotlin/dev/typetype/server/TestDatabase.kt (98%) rename {src => server-http/src}/main/kotlin/dev/typetype/server/CompressionConfig.kt (100%) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt rename {src => server-http/src}/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/CommentRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SearchRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/StreamRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt (100%) rename {src/test/kotlin/dev/typetype/server => server-http/src/test/kotlin/dev/typetype/server/services}/SabrPlaybackGranularRoutesTest.kt (100%) rename {src => server-http/src}/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt (100%) rename {src => server-playback/src}/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt (100%) delete mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt rename {src => server-portability/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt (100%) rename {src => server-portability/src}/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ManifestServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SearchContentKindTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/UrlValidatorTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt (100%) rename {src => server-services/src}/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt (100%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt (100%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/FakeChannelService.kt (100%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt (98%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt (98%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt (100%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt (100%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt (96%) rename {src/test => server-services/src/testFixtures}/kotlin/dev/typetype/server/TestCommentFixtures.kt (100%) create mode 100644 server-test-support/build.gradle.kts rename {src/test/kotlin/dev/typetype/server/services => server-test-support/src/main/kotlin/dev/typetype/server}/SabrProbeConfig.kt (98%) rename {src/test/kotlin/dev/typetype/server/services => server-test-support/src/main/kotlin/dev/typetype/server}/SabrProbeDiagnostics.kt (84%) rename {src/test/kotlin/dev/typetype/server/services => server-test-support/src/main/kotlin/dev/typetype/server}/SabrProbeFetch.kt (91%) rename {src/test/kotlin/dev/typetype/server/services => server-test-support/src/main/kotlin/dev/typetype/server}/SabrProbeFetchResult.kt (87%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt (100%) rename {src => server-token-gateway/src}/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt (100%) diff --git a/build.gradle.kts b/build.gradle.kts index becc6cba..eeffcaa7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -66,6 +66,11 @@ dependencies { implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.4.0") + testImplementation(project(":server-test-support")) + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("com.password4j:password4j:1.8.4") testRuntimeOnly("org.junit.platform:junit-platform-launcher") @@ -103,19 +108,6 @@ tasks.jacocoTestReport { } } -tasks.jacocoTestCoverageVerification { - dependsOn(tasks.test) - violationRules { - rule { - limit { - counter = "LINE" - value = "COVEREDRATIO" - minimum = "0.20".toBigDecimal() - } - } - } -} - val verifySabrBoundary = tasks.register("verifySabrBoundary") { doLast { val adapterRoot = file("src/main/kotlin/dev/typetype/server/sabr") @@ -137,7 +129,6 @@ val verifySabrBoundary = tasks.register("verifySabrBoundary") { } tasks.check { - dependsOn(tasks.jacocoTestCoverageVerification) dependsOn(verifySabrBoundary) } diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts index 1ba3a81d..f0fe8083 100644 --- a/server-admin/build.gradle.kts +++ b/server-admin/build.gradle.kts @@ -22,8 +22,24 @@ dependencies { implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("org.slf4j:slf4j-api:2.0.16") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminManagedAccessRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/AdminUserAccessModeRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt b/server-admin/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt rename to server-admin/src/test/kotlin/dev/typetype/server/InternalObservabilityRoutesTest.kt diff --git a/server-auth/build.gradle.kts b/server-auth/build.gradle.kts index 40d9fa0e..72c21688 100644 --- a/server-auth/build.gradle.kts +++ b/server-auth/build.gradle.kts @@ -11,6 +11,11 @@ dependencies { implementation("com.password4j:password4j:1.8.4") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(project(":server-test-support")) testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } diff --git a/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AccountIdentityServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AuthCookieHelpersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/AvatarServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt b/server-auth/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt rename to server-auth/src/test/kotlin/dev/typetype/server/OidcAuthServiceTest.kt diff --git a/server-cache/build.gradle.kts b/server-cache/build.gradle.kts index 8c743aa1..8b432e73 100644 --- a/server-cache/build.gradle.kts +++ b/server-cache/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + `java-test-fixtures` kotlin("jvm") kotlin("plugin.serialization") } @@ -7,8 +8,17 @@ dependencies { api("io.lettuce:lettuce-core:7.7.0.RELEASE") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + testImplementation(testFixtures(project(":server-cache"))) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/test/kotlin/dev/typetype/server/FakeCacheService.kt b/server-cache/src/testFixtures/kotlin/dev/typetype/server/FakeCacheService.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeCacheService.kt rename to server-cache/src/testFixtures/kotlin/dev/typetype/server/FakeCacheService.kt diff --git a/server-core/build.gradle.kts b/server-core/build.gradle.kts index 0514878c..b4b4830b 100644 --- a/server-core/build.gradle.kts +++ b/server-core/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + `java-test-fixtures` kotlin("jvm") kotlin("plugin.serialization") } @@ -6,8 +7,17 @@ plugins { dependencies { api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + testImplementation(testFixtures(project(":server-core"))) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt b/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt new file mode 100644 index 00000000..e8aa885c --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/services/AdminSettingsCache.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AdminSettingsItem + +object AdminSettingsCache { + @Volatile + private var cachedSettings: AdminSettingsItem? = null + + fun get(): AdminSettingsItem? = cachedSettings + + fun set(value: AdminSettingsItem?) { + cachedSettings = value + } + + fun clear() { + cachedSettings = null + } +} diff --git a/src/test/kotlin/dev/typetype/server/TestFixtures.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestFixtures.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/TestStreamResponseFixture.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestStreamResponseFixture.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestStreamResponseFixture.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestStreamResponseFixture.kt diff --git a/src/test/kotlin/dev/typetype/server/TestVideoItemFixture.kt b/server-core/src/testFixtures/kotlin/dev/typetype/server/TestVideoItemFixture.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestVideoItemFixture.kt rename to server-core/src/testFixtures/kotlin/dev/typetype/server/TestVideoItemFixture.kt diff --git a/server-db/build.gradle.kts b/server-db/build.gradle.kts index 7b90222f..8b78c3c4 100644 --- a/server-db/build.gradle.kts +++ b/server-db/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + `java-test-fixtures` kotlin("jvm") } @@ -9,8 +10,22 @@ dependencies { api("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.4.0") + testImplementation("org.testcontainers:testcontainers:2.0.5") + testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation(testFixtures(project(":server-db"))) + "testFixturesImplementation"("org.testcontainers:testcontainers:2.0.5") + "testFixturesImplementation"("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt b/server-db/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt rename to server-db/src/test/kotlin/dev/typetype/server/DatabaseCollectionMetadataMigrationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt b/server-db/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt rename to server-db/src/test/kotlin/dev/typetype/server/DatabaseImportedMediaRepairMigrationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/TestDatabase.kt rename to server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt index 013b07cd..0cdf86d2 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.AdminSettingsCache import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.AllowedPlaylistsTable import dev.typetype.server.db.tables.BlockedChannelsTable @@ -49,7 +50,6 @@ import dev.typetype.server.db.tables.RecommendationOnboardingStateTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.db.tables.ProfileAccountsTable import dev.typetype.server.db.tables.PresenceKeysTable -import dev.typetype.server.services.AdminSettingsService import org.jetbrains.exposed.v1.jdbc.deleteAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.testcontainers.containers.ContainerLaunchException @@ -71,7 +71,7 @@ object TestDatabase { } fun setup() { - AdminSettingsService.clearCache() + AdminSettingsCache.clear() if (initialized) return synchronized(this) { if (initialized) return @@ -158,6 +158,6 @@ object TestDatabase { RecommendationOnboardingStateTable.deleteAll() ProfileAccountsTable.deleteAll() PresenceKeysTable.deleteAll() - AdminSettingsService.clearCache() + AdminSettingsCache.clear() } } diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts index 24fddfdf..b9b571aa 100644 --- a/server-domain/build.gradle.kts +++ b/server-domain/build.gradle.kts @@ -12,8 +12,16 @@ dependencies { implementation(project(":server-db")) implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts index a23eee47..2f5a0a7b 100644 --- a/server-http/build.gradle.kts +++ b/server-http/build.gradle.kts @@ -19,14 +19,34 @@ dependencies { implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("io.ktor:ktor-server-rate-limit-jvm:3.5.2") + implementation("io.ktor:ktor-server-status-pages-jvm:3.5.2") + implementation("io.ktor:ktor-server-compression-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-compression-jvm:3.5.2") + implementation("io.ktor:ktor-server-call-logging-jvm:3.5.2") implementation("io.ktor:ktor-utils-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/main/kotlin/dev/typetype/server/CompressionConfig.kt b/server-http/src/main/kotlin/dev/typetype/server/CompressionConfig.kt similarity index 100% rename from src/main/kotlin/dev/typetype/server/CompressionConfig.kt rename to server-http/src/main/kotlin/dev/typetype/server/CompressionConfig.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt new file mode 100644 index 00000000..5230ec02 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt @@ -0,0 +1,43 @@ +package dev.typetype.server + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.routes.TooManyRequestsBodyAttribute +import dev.typetype.server.routes.isMultipartSizeLimit +import dev.typetype.server.routes.respondPortabilityError +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.Application +import io.ktor.server.application.ApplicationCall +import io.ktor.server.application.install +import io.ktor.server.plugins.statuspages.StatusPages +import io.ktor.server.request.path +import io.ktor.server.response.respond +import kotlinx.coroutines.CancellationException +import org.slf4j.LoggerFactory + +fun Application.configureStatusPages() { + val log = LoggerFactory.getLogger("RequestLogger") + install(StatusPages) { + status(HttpStatusCode.TooManyRequests) { call, status -> + if (call.attributes.contains(TooManyRequestsBodyAttribute)) return@status + if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") + call.respond(status, ErrorResponse("Too many requests", "rate_limited")) + } + exception { call, cause -> + log.warn("Bad request: ${cause.message}") + call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request", "bad_request")) + } + exception { call, cause -> + if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception + if (cause is kotlinx.coroutines.CancellationException) throw cause + // Ktor's multipart producer can fail outside the route's receive block. + if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) { + call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException()) + return@exception + } + log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) + call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) + } + } +} + diff --git a/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AccountProfilesRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AllowedChannelsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyByteRangeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyPlayableContractRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlyProgressiveContractRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceOpenRangeRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/AudioOnlySourceRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BlockedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BugReportRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/BulletCommentRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ChannelRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/CommentRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DeArrowThumbnailContentTypeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayArtifactProxyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/DownloaderGatewayRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt b/server-http/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt rename to server-http/src/test/kotlin/dev/typetype/server/FakeRemoteBrowserClient.kt diff --git a/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/FavoritesRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HistoryRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ManifestRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/NicoVideoProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/NotificationsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/OidcAuthRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PodcastRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityArchiveReproductionTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityMultipartFailureTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PortabilityRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PresenceRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProfileRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProgressRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProviderMediaHandleRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/ProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/PublicPlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/RestoreRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrHttpRandomAccessInitProbeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrMediaResponseWriterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrRoutesAccessTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SabrStreamContractFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SavedPlaylistRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SearchHistoryRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SearchRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/StoryboardProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionShortsFeedRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsCanonicalizationRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsDeleteProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SubscriptionsRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/SuggestionRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/TrendingRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/WatchLaterRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YouTubeSubtitleProxyRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserCompleteRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeSessionRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackContinuityProbeTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveRoundedBoundaryWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAccessValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackAudioOnlyWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRateTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackRecoveryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackSeekRouteTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt diff --git a/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SabrPlaybackGranularRoutesTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt rename to server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackLiveGapServiceTest.kt diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts index ae42b1cc..acd7588b 100644 --- a/server-playback/build.gradle.kts +++ b/server-playback/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + `java-test-fixtures` kotlin("jvm") kotlin("plugin.serialization") } @@ -19,6 +20,9 @@ dependencies { implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("org.slf4j:slf4j-api:2.0.16") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(project(":server-test-support")) testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt rename to server-playback/src/test/kotlin/dev/typetype/server/YoutubeSessionTokenScopeTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt deleted file mode 100644 index 868767af..00000000 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/TestStreamResponseFixture.kt +++ /dev/null @@ -1,53 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.models.AudioStreamItem -import dev.typetype.server.models.StreamResponse -import dev.typetype.server.models.VideoStreamItem - -fun testStreamResponse( - videoOnlyStreams: List = listOf(playbackTestVideoStream()), - audioStreams: List = listOf(playbackTestAudioStream()), - duration: Long = 300, - hlsUrl: String = "", - dashMpdUrl: String = "", -): StreamResponse = StreamResponse( - id = "test-id", - title = "Test Video", - uploaderName = "Test Channel", - uploaderUrl = "https://youtube.com/channel/test", - uploaderAvatarUrl = "", - thumbnailUrl = "", - description = "", - duration = duration, - viewCount = 0, - likeCount = 0, - dislikeCount = 0, - uploadDate = "", - uploaded = 0, - uploaderSubscriberCount = 0, - uploaderVerified = false, - category = "", - license = "", - visibility = "", - tags = emptyList(), - streamType = "video_stream", - isLive = false, - isPostLive = false, - isLiveContent = false, - hasLiveManifest = false, - isShortFormContent = false, - requiresMembership = false, - startPosition = 0L, - streamSegments = emptyList(), - hlsUrl = hlsUrl, - dashMpdUrl = dashMpdUrl, - videoStreams = emptyList(), - audioStreams = audioStreams, - originalAudioTrackId = null, - preferredDefaultAudioTrackId = null, - videoOnlyStreams = videoOnlyStreams, - subtitles = emptyList(), - previewFrames = emptyList(), - sponsorBlockSegments = emptyList(), - relatedStreams = emptyList(), -) diff --git a/server-portability/build.gradle.kts b/server-portability/build.gradle.kts index 2b6b19b7..c46ef7c2 100644 --- a/server-portability/build.gradle.kts +++ b/server-portability/build.gradle.kts @@ -14,8 +14,20 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind:2.22.2") implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { jvmToolchain(25) } + +tasks.test { + useJUnitPlatform() +} diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutActivitySignalServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutImportProgressTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutIssueServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/YoutubeTakeoutPathHintsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/FlowPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/GrayjayPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/InvidiousPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/LibreTubePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/MaterialiousPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/NewPipePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/OpmlPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PipedPortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityEngineTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityJobStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilityProgressTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/PortabilitySpoolTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityHistoryImportTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/ViewTubePortabilityAdapterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutHtmlPortabilityReaderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt rename to server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutPortabilityAdapterTest.kt diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts index b33063bc..bbcd7419 100644 --- a/server-sabr/build.gradle.kts +++ b/server-sabr/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + api("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts index 6e2d8339..befc6f34 100644 --- a/server-services/build.gradle.kts +++ b/server-services/build.gradle.kts @@ -1,5 +1,6 @@ import java.time.Instant plugins { + `java-test-fixtures` kotlin("jvm") kotlin("plugin.serialization") } @@ -30,6 +31,27 @@ dependencies { implementation("org.jetbrains.exposed:exposed-core:1.5.0") implementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") implementation("io.lettuce:lettuce-core:7.7.0.RELEASE") + testImplementation("org.testcontainers:testcontainers:2.0.5") + testImplementation("org.testcontainers:testcontainers-postgresql:2.0.5") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + testImplementation(testFixtures(project(":server-db"))) + testImplementation(testFixtures(project(":server-services"))) + testImplementation(testFixtures(project(":server-core"))) + testImplementation(testFixtures(project(":server-cache"))) + testFixturesImplementation(project(":server-cache")) + testFixturesImplementation(project(":server-db")) + testFixturesImplementation(testFixtures(project(":server-db"))) + testFixturesImplementation("org.jetbrains.exposed:exposed-core:1.5.0") + testFixturesImplementation("org.jetbrains.exposed:exposed-jdbc:1.5.0") + testFixturesImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") +"testFixturesImplementation"("io.lettuce:lettuce-core:7.7.0.RELEASE") + testImplementation(project(":server-test-support")) + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") + testImplementation("io.mockk:mockk:1.14.11") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } kotlin { @@ -79,3 +101,7 @@ val generateBuildInfo = tasks.register("generateBuildInfo") { sourceSets.named("main") { kotlin.srcDir(generatedBuildInfoDir) } tasks.named("compileKotlin") { dependsOn(generateBuildInfo) } + +tasks.test { + useJUnitPlatform() +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt index 7a761a05..407f123a 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt @@ -17,7 +17,7 @@ class AdminSettingsService( ) { suspend fun get(): AdminSettingsItem { - cachedSettings?.let { return it } + AdminSettingsCache.get()?.let { return it } val settings = DatabaseFactory.query { AdminSettingsTable.selectAll().singleOrNull()?.let { AdminSettingsItem( @@ -43,7 +43,7 @@ class AdminSettingsService( ).normalized() } ?: defaultSettings().normalized() } - cachedSettings = settings + AdminSettingsCache.set(settings) return settings } @@ -98,7 +98,7 @@ class AdminSettingsService( } } } - cachedSettings = settings + AdminSettingsCache.set(settings) return settings } @@ -141,11 +141,8 @@ class AdminSettingsService( } companion object { - @Volatile - private var cachedSettings: AdminSettingsItem? = null - fun clearCache() { - cachedSettings = null + AdminSettingsCache.clear() } } } diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/BilibiliRelatedServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CachedStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CachedSuggestionServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/CustomAvatarServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ExtractionErrorSanitizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HistoryProgressMapperTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCandidateServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationCursorCodecTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationLiveTitleDetectorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationMixerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolRotationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsDeduplicatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsFallbackTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationShortsRefresherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt b/server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/HomeRecommendationTemporalBoostTest.kt diff --git a/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt b/server-services/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/LiveStreamTypesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ManifestRepresentationIdsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ManifestServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt b/server-services/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/NativeManifestUtilsTest.kt diff --git a/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/NicoVideoProxyServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupSqliteReaderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PipePipeBackupTimelineNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PlaylistServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ProviderMediaHandleServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PublicCachePolicyTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PublishedAtMapperTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushDeviceRegistryTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushNotificationDeliveryStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt b/server-services/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/PushNotificationSupportTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RequestObservabilityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RetryingProxyInputStreamTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssDocumentRendererTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt b/server-services/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SearchContentKindTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SearchFilterMappersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StoryboardProxyDetectorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamAudioContractResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt diff --git a/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedAvailabilityTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedCacheKeysTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedOrdererTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSelectionStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionFeedSnapshotStoreTest.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/TypeTypeBackupServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/UnifiedPushEndpointValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionHlsManifestServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeSessionStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/YoutubeTakeoutParserServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/HomeWarmupTrackerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/OpenMojiProxyServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/RelatedItemMappersTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/SabrAdaptiveInitializationTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClientTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/TypetypeYoutubeSessionPoTokenProviderTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContentFetcherTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleContractTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientFallbackStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubePlayerClientStreamServiceTest.kt diff --git a/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt rename to server-services/src/test/kotlin/dev/typetype/server/services/YoutubeTakeoutImportCacheTest.kt diff --git a/src/test/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/ActiveSessionTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/FakeChannelService.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/FakeChannelService.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/FakeChannelService.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/FakeChannelService.kt diff --git a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt index 0d65cdde..6237f6dc 100644 --- a/src/test/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/GitHubIssueServiceTestReports.kt @@ -4,7 +4,7 @@ import dev.typetype.server.models.AdminBugReportDetailResponse import dev.typetype.server.models.BugApiErrorItem import dev.typetype.server.models.BugReportContextItem -internal object GitHubIssueServiceTestReports { +object GitHubIssueServiceTestReports { fun sampleReport(): AdminBugReportDetailResponse = AdminBugReportDetailResponse( id = "report-id", category = "player", diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt index 73902f88..601a2096 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationItemFixtures.kt @@ -8,7 +8,7 @@ import dev.typetype.server.services.HomeRecommendationSessionIntent import dev.typetype.server.services.HomeRecommendationSourceTag import dev.typetype.server.services.HomeRecommendationTaggedVideo -internal object HomeRecommendationItemFixtures { +object HomeRecommendationItemFixtures { val context: HomeRecommendationSessionContext = HomeRecommendationSessionContext( intent = HomeRecommendationSessionIntent.AUTO, deviceClass = HomeRecommendationDeviceClass.UNKNOWN, diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/HomeRecommendationTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/PipePipeBackupTestFixtures.kt diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt similarity index 96% rename from src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt index a44151fb..44501ded 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt +++ b/server-services/src/testFixtures/kotlin/dev/typetype/server/SubscriptionFeedTestFixtures.kt @@ -5,7 +5,7 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.models.VideoItem -internal object SubscriptionFeedTestFixtures { +object SubscriptionFeedTestFixtures { fun video( uploaded: Long, channel: String = "Ch", diff --git a/src/test/kotlin/dev/typetype/server/TestCommentFixtures.kt b/server-services/src/testFixtures/kotlin/dev/typetype/server/TestCommentFixtures.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/TestCommentFixtures.kt rename to server-services/src/testFixtures/kotlin/dev/typetype/server/TestCommentFixtures.kt diff --git a/server-test-support/build.gradle.kts b/server-test-support/build.gradle.kts new file mode 100644 index 00000000..a7327d7c --- /dev/null +++ b/server-test-support/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + kotlin("jvm") +} + +dependencies { + api(project(":server-core")) + api(project(":server-db")) + api(project(":server-cache")) + api("org.testcontainers:testcontainers:2.0.5") + api("org.testcontainers:testcontainers-postgresql:2.0.5") + api("io.mockk:mockk:1.14.11") + api("org.junit.jupiter:junit-jupiter:6.1.3") + api("io.ktor:ktor-utils-jvm:3.5.2") + api("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + api("io.ktor:ktor-server-test-host-jvm:3.5.2") + api("io.ktor:ktor-server-content-negotiation-jvm:3.5.2") + api("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") + api(project(":server-services")) + api(project(":server-playback")) + api(project(":server-token-gateway")) + api(project(":server-sabr")) +} + +kotlin { + jvmToolchain(25) +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt similarity index 98% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt index 2ce9c2a1..0efabf12 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeConfig.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeConfig.kt @@ -1,5 +1,7 @@ package dev.typetype.server.services +import dev.typetype.server.sabr.SabrProtocolException +import dev.typetype.server.sabr.SabrRecoverableException import io.mockk.every import io.mockk.coEvery import io.mockk.coVerify diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt similarity index 84% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt index f4981740..3bb64d82 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeDiagnostics.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeDiagnostics.kt @@ -6,30 +6,30 @@ private const val DEFAULT_SABR_PROBE_VIDEO_ITAG = 137 private const val DEFAULT_SABR_PROBE_AUDIO_ITAG = 140 private const val DEFAULT_SABR_PROBE_TIMEOUT_MS = 60_000L -internal fun sabrProbeTokenServiceUrl(): String = +fun sabrProbeTokenServiceUrl(): String = envValue("SUBTITLE_SERVICE_URL") ?: "http://localhost:8081" -internal fun sabrProbeVideoId(): String = +fun sabrProbeVideoId(): String = envValue("SABR_PROBE_VIDEO") ?: envValues("SABR_PROBE_VIDEOS").firstOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ID -internal fun sabrProbeVideoIds(): List = +fun sabrProbeVideoIds(): List = envValues("SABR_PROBE_VIDEOS").ifEmpty { listOf(sabrProbeVideoId()) } -internal fun sabrProbePlayerTimeMs(): Long = +fun sabrProbePlayerTimeMs(): Long = envValue("SABR_PROBE_PLAYER_TIME_MS")?.toLongOrNull()?.takeIf { it >= 0L } ?: DEFAULT_SABR_PROBE_PLAYER_TIME_MS -internal fun sabrProbeTimeoutMs(): Long = +fun sabrProbeTimeoutMs(): Long = envValue("SABR_PROBE_FETCH_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } ?: envValue("SABR_PROBE_TIMEOUT_MS")?.toLongOrNull()?.takeIf { it > 0L } ?: DEFAULT_SABR_PROBE_TIMEOUT_MS -internal fun sabrProbeAudioItag(): Int = +fun sabrProbeAudioItag(): Int = envValue("SABR_PROBE_AUDIO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_AUDIO_ITAG -internal fun sabrProbeVideoItags(): List { +fun sabrProbeVideoItags(): List { val explicit = envIntValues("SABR_PROBE_VIDEO_ITAGS") val primary = envValue("SABR_PROBE_VIDEO_ITAG")?.toIntOrNull() ?: DEFAULT_SABR_PROBE_VIDEO_ITAG val base = explicit.ifEmpty { listOf(primary) } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt similarity index 91% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt index 7c28b7f1..1150edaf 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetch.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetch.kt @@ -4,7 +4,7 @@ import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat -internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { +fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Unit { println( "$label format itag=${format.itag} audio=${format.isAudio} video=${format.isVideo} " + "size=${format.width}x${format.height} bitrate=${format.bitrate} mime=${format.mimeType} " + @@ -14,7 +14,7 @@ internal fun printSabrProbeFormat(label: String, format: YoutubeSabrFormat): Uni ) } -internal fun printSabrProbeFetch( +fun printSabrProbeFetch( label: String, holder: SabrSessionHolder, request: SabrSegmentRequest, @@ -34,7 +34,7 @@ internal fun printSabrProbeFetch( result.error?.let { println("$label error ${it.javaClass.simpleName}: ${it.message}") } } -internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { +fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { val header = segment.header return "headerId=${header.headerId} videoId=${header.videoId} itag=${header.itag} " + "seq=${header.sequenceNumber} init=${header.isInitSegment} startMs=${header.startMs} " + @@ -42,7 +42,7 @@ internal fun sabrProbeSegmentHeader(segment: SabrMediaSegment): String { "contentLength=${header.contentLength} bytes=${segment.length} summary=${header.summarize()}" } -internal fun sabrProbeRequestSummary( +fun sabrProbeRequestSummary( holder: SabrSessionHolder, request: SabrSegmentRequest, ): String { diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt similarity index 87% rename from src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt rename to server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt index 86f2cf76..8a9f6272 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeFetchResult.kt +++ b/server-test-support/src/main/kotlin/dev/typetype/server/SabrProbeFetchResult.kt @@ -5,7 +5,14 @@ import kotlinx.coroutines.withTimeoutOrNull import dev.typetype.server.sabr.SabrMediaSegment import dev.typetype.server.sabr.SabrSegmentRequest -internal suspend fun fetchSabrProbeSegment( +data class SabrProbeFetchResult( + val segment: SabrMediaSegment?, + val elapsedMs: Long, + val timedOut: Boolean, + val error: Throwable?, +) + +suspend fun fetchSabrProbeSegment( store: SabrSessionStore, holder: SabrSessionHolder, request: SabrSegmentRequest, diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts index e1e63a83..9bb1e6b1 100644 --- a/server-token-gateway/build.gradle.kts +++ b/server-token-gateway/build.gradle.kts @@ -17,6 +17,8 @@ dependencies { implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") + testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") + testImplementation("io.ktor:ktor-serialization-kotlinx-json-jvm:3.5.2") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeRemoteBrowserMessageGuardTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCookieNormalizerTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCredentialValidatorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt similarity index 100% rename from src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt rename to server-token-gateway/src/test/kotlin/dev/typetype/server/YoutubeSessionCryptoTest.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index 7caea1eb..86c5ea82 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -19,6 +19,7 @@ include(":server-http") include(":server-playback") include(":server-portability") include(":server-sabr") +include(":server-test-support") include(":server-services") include(":server-token-gateway") diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index 85f61617..a092e265 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -21,6 +21,7 @@ import io.ktor.server.request.path import io.ktor.server.response.respond import io.ktor.server.websocket.WebSockets import io.ktor.util.AttributeKey +import dev.typetype.server.configureStatusPages import dev.typetype.server.routes.TooManyRequestsBodyAttribute import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory @@ -96,28 +97,3 @@ fun Application.configurePlugins(authService: AuthService) { -internal fun Application.configureStatusPages() { - val log = LoggerFactory.getLogger("RequestLogger") - install(StatusPages) { - status(HttpStatusCode.TooManyRequests) { call, status -> - if (call.attributes.contains(TooManyRequestsBodyAttribute)) return@status - if (!call.response.headers.contains(HttpHeaders.RetryAfter)) call.response.headers.append(HttpHeaders.RetryAfter, "60") - call.respond(status, ErrorResponse("Too many requests", "rate_limited")) - } - exception { call, cause -> - log.warn("Bad request: ${cause.message}") - call.respond(HttpStatusCode.BadRequest, ErrorResponse(cause.message ?: "Bad request", "bad_request")) - } - exception { call, cause -> - if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception - if (cause is kotlinx.coroutines.CancellationException) throw cause - // Ktor's multipart producer can fail outside the route's receive block. - if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) { - call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException()) - return@exception - } - log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause) - call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error")) - } - } -} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt index 55d799d9..d82d9350 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrProbeTest.kt @@ -1,10 +1,3 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrMediaSegment - -internal data class SabrProbeFetchResult( - val segment: SabrMediaSegment?, - val elapsedMs: Long, - val timedOut: Boolean, - val error: Throwable?, -) From d2a9f1e84d92213100c5cc8134217609032de52e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 19 Sep 2026 10:13:17 +0200 Subject: [PATCH 044/137] ci: route server workflows to CI stack --- .github/workflows/ci.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/docker.yml | 8 ++++---- .github/workflows/openapi.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a22891d..5f5edc8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ concurrency: jobs: build: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} + runs-on: [self-hosted, Linux, X64, ci-stack] steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 18d1dfd2..79fbc5bf 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ concurrency: jobs: coverage: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} + runs-on: [self-hosted, Linux, X64, ci-stack] steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a53dc8d6..a2f9b90c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ concurrency: jobs: prepare-image: - runs-on: [self-hosted, Linux, X64, r730] + runs-on: [self-hosted, Linux, X64, ci-stack] permissions: contents: read outputs: @@ -76,7 +76,7 @@ jobs: build-platform: needs: prepare-image - runs-on: [self-hosted, Linux, X64, r730, docker] + runs-on: [self-hosted, Linux, X64, ci-stack, docker] timeout-minutes: 20 permissions: contents: read @@ -150,7 +150,7 @@ jobs: build-and-push: needs: [prepare-image, build-platform] - runs-on: [self-hosted, Linux, X64, r730, docker] + runs-on: [self-hosted, Linux, X64, ci-stack, docker] timeout-minutes: 10 permissions: contents: read @@ -235,7 +235,7 @@ jobs: notify-orchestrator: needs: [prepare-image, build-and-push] if: github.ref_name == 'dev' || github.ref_name == 'main' || startsWith(github.ref, 'refs/tags/v') - runs-on: [self-hosted, Linux, X64, r730] + runs-on: [self-hosted, Linux, X64, ci-stack] permissions: contents: read env: diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 4b50635e..1369962f 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -15,7 +15,7 @@ concurrency: jobs: validate: - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }} + runs-on: [self-hosted, Linux, X64, ci-stack] steps: - uses: actions/checkout@v7 - name: Isolate Gradle user home From d861cdfc911cbb70e0230f164d377d4201d0b32b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 19 Sep 2026 15:37:02 +0000 Subject: [PATCH 045/137] fix: isolate optional subtitle failures --- .../dev/typetype/server/ExtractionServiceRegistry.kt | 11 ++++++++--- .../typetype/server/services/PipePipeStreamService.kt | 10 +++++++++- .../server/services/YouTubeSubtitleDeliveryService.kt | 9 +++++++-- .../services/YouTubeSubtitleDeliveryServiceTest.kt | 10 ++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index d2f72dc5..135d47a1 100644 --- a/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -74,6 +74,11 @@ internal class ExtractionServiceRegistry( val httpClient = OkHttpClient.Builder() .apply { youtubeProxySelector?.let(::proxySelector) } .build() + val subtitleHttpClient = httpClient.newBuilder() + .connectTimeout(5, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .callTimeout(15, TimeUnit.SECONDS) + .build() val proxyHttpClient: OkHttpClient = httpClient.newBuilder() .dispatcher(proxyDispatcher()) .connectionPool(ConnectionPool(64, 5, TimeUnit.MINUTES)) @@ -82,7 +87,7 @@ internal class ExtractionServiceRegistry( .followRedirects(true) .build() val sabrSessionStore = SabrSessionStore(subtitleServiceUrl, initCache = cache) - val youtubeSubtitleService = YouTubeSubtitleService(httpClient, subtitleServiceUrl) + val youtubeSubtitleService = YouTubeSubtitleService(subtitleHttpClient, subtitleServiceUrl) private val bilibiliRelatedService = BilibiliRelatedService() private val directPipePipeStreamService = PipePipeStreamService( cache, @@ -130,9 +135,9 @@ internal class ExtractionServiceRegistry( val youtubeSubtitleDeliveryService = YouTubeSubtitleDeliveryService( StreamYouTubeSubtitleResolver(youtubeSabrStreamService, youtubeSubtitleService::fetchSubtitleInventory), TokenYouTubeSubtitleContentFetcher( - httpClient, + subtitleHttpClient, subtitleServiceUrl, - OkHttpYouTubeSubtitleContentFetcher(httpClient), + OkHttpYouTubeSubtitleContentFetcher(subtitleHttpClient), ), YouTubeSubtitleCache(cache), ) diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index 85209166..ceb8084f 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -5,6 +5,7 @@ import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.SponsorBlockSegmentItem import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -56,7 +57,14 @@ internal class PipePipeStreamService( streamInfo.setSponsorBlockSegments(segmentsDeferred.await()) val response = StreamAudioContractResolver.apply(streamInfo.toStreamResponse()) val withSubtitles = if (response.subtitles.isEmpty() && service.serviceId == 0) { - response.copy(subtitles = subtitleService.fetchSubtitles(streamInfo.id)) + val subtitles = try { + subtitleService.fetchSubtitles(streamInfo.id) + } catch (error: CancellationException) { + throw error + } catch (_: Exception) { + emptyList() + } + response.copy(subtitles = subtitles) } else { response } diff --git a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt index 1e5ff2a2..b097ebce 100644 --- a/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryService.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CancellationException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Semaphore @@ -28,8 +29,12 @@ internal class YouTubeSubtitleDeliveryService( pending.complete(result) result } catch (error: Throwable) { - pending.completeExceptionally(error) - throw error + if (error is CancellationException) { + pending.cancel(error) + throw error + } + pending.complete(YouTubeSubtitleContentResult.Unavailable) + YouTubeSubtitleContentResult.Unavailable } finally { inFlight.remove(selection.cacheKey, pending) upstreamPermits.release() diff --git a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt index b09e96ca..c04a60fa 100644 --- a/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/YouTubeSubtitleDeliveryServiceTest.kt @@ -122,6 +122,16 @@ class YouTubeSubtitleDeliveryServiceTest { assertEquals(0, fetches.get()) } + @Test + fun `upstream subtitle failure is isolated as unavailable`() = runTest { + val service = service( + resolver = { readyTrack() }, + fetcher = { _, _ -> error("ReCaptcha required") }, + ) + + assertEquals(YouTubeSubtitleContentResult.Unavailable, service.fetch(SELECTION)) + } + private fun service( resolver: suspend (YouTubeSubtitleSelection) -> YouTubeSubtitleResolution, fetcher: suspend (String, YouTubeSubtitleFormat) -> YouTubeSubtitleFetchResult, From d9e07ba3ce9f33ed0e4296e5043747a8ccab7514 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 19 Sep 2026 15:37:02 +0000 Subject: [PATCH 046/137] fix: bound nonstreaming proxy requests --- .../typetype/server/services/OkHttpProxyService.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 9b906a23..e1484adf 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -11,6 +11,7 @@ import okhttp3.Request import java.io.ByteArrayInputStream import java.net.URLEncoder import java.nio.charset.StandardCharsets +import java.util.concurrent.TimeUnit internal val GOOGLEVIDEO_URL_REGEX = Regex("""https://[a-z0-9.\-]+\.googlevideo\.com/\S+""") private val CPN_TRACKING_PARAM_REGEX = Regex("[&?]cpn=[^&]*") @@ -30,6 +31,11 @@ class OkHttpProxyService( private val mediaHandleService: ProviderMediaHandleService? = null, ) : ProxyService, ProviderMediaAwareProxyService { private val executor = ProxyHttpExecutor(client) + private val boundedExecutor = ProxyHttpExecutor( + client.newBuilder() + .callTimeout(30, TimeUnit.SECONDS) + .build(), + ) override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = pipeInternal(url, rangeHeader, domandBid, providerMediaManifest = false) @@ -53,6 +59,7 @@ class OkHttpProxyService( val fragment = if (hashIdx >= 0) url.substring(hashIdx + 1) else "" val resolvedDomandBid = domandBid ?: if (fragment.isNotBlank()) parseNicoCookie(fragment) else null validateProxyUrl(fetchUrl)?.let { return@withContext ExtractionResult.BadRequest(it) } + val requestExecutor = if (rangeHeader == null) boundedExecutor else executor runCatching { val cleanUrl = stripTrackingParams(fetchUrl) val bilibili = isBilibili(cleanUrl) @@ -67,9 +74,9 @@ class OkHttpProxyService( if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() if (bilibili && rangeHeader != null) { - return@withContext readBilibiliRangeWithRetry(executor::execute, request, requestContext::ensureActive) + return@withContext readBilibiliRangeWithRetry(requestExecutor::execute, request, requestContext::ensureActive) } - executor.execute(request) + requestExecutor.execute(request) }.fold( onSuccess = { response -> val body = response.body From 1e83eda74ca8917d5b3a3a60193c6bb33477a9c6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 19 Sep 2026 16:58:22 +0000 Subject: [PATCH 047/137] feat: persist autoplay countdown setting --- openapi/components/access-control.yaml | 5 +++++ .../server/db/SettingsSchemaMigrations.kt | 1 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../services/SettingsPersistenceMappers.kt | 3 +++ .../SettingsAutoplayOnOpenRoutesTest.kt | 19 +++++++++++++++++++ 6 files changed, 30 insertions(+) diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 9c0d05bc..5e72fa9b 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -12,6 +12,11 @@ SettingsItem: defaultLandingPage: { type: string, default: home } autoplay: { type: boolean, default: true } autoplayOnOpen: { type: boolean, default: true } + autoplayCountdownSeconds: + type: integer + minimum: 0 + maximum: 60 + default: 10 volume: { type: number, format: double, default: 1.0 } muted: { type: boolean, default: false } notificationPopupsEnabled: { type: boolean, default: true } diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index b6152524..2a0338c8 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -31,6 +31,7 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_on_open BOOLEAN NOT NULL DEFAULT true") exec("UPDATE settings SET autoplay_on_open = autoplay") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_countdown_seconds INTEGER NOT NULL DEFAULT 10") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_landing_page TEXT NOT NULL DEFAULT 'home'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index f540816c..8fbd1523 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -10,6 +10,7 @@ object SettingsTable : Table("settings") { val defaultLandingPage = text("default_landing_page").default("home") val autoplay = bool("autoplay").default(true) val autoplayOnOpen = bool("autoplay_on_open").default(true) + val autoplayCountdownSeconds = integer("autoplay_countdown_seconds").default(10) val skipPlaylistAutoplayScreen = bool("skip_playlist_autoplay_screen").default(false) val volume = double("volume").default(1.0) val muted = bool("muted").default(false) diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 04eb8e49..40d241f7 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -10,6 +10,7 @@ data class SettingsItem( val defaultLandingPage: String = "home", val autoplay: Boolean = true, val autoplayOnOpen: Boolean = true, + val autoplayCountdownSeconds: Int = 10, val skipPlaylistAutoplayScreen: Boolean = false, val volume: Double = 1.0, val muted: Boolean = false, diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 8963c806..64014184 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -23,6 +23,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultLandingPage = this[SettingsTable.defaultLandingPage], autoplay = this[SettingsTable.autoplay], autoplayOnOpen = this[SettingsTable.autoplayOnOpen], + autoplayCountdownSeconds = this[SettingsTable.autoplayCountdownSeconds], skipPlaylistAutoplayScreen = this[SettingsTable.skipPlaylistAutoplayScreen], volume = this[SettingsTable.volume], muted = this[SettingsTable.muted], @@ -64,6 +65,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultLandingPage] = settings.defaultLandingPage this[SettingsTable.autoplay] = settings.autoplay this[SettingsTable.autoplayOnOpen] = settings.autoplayOnOpen + this[SettingsTable.autoplayCountdownSeconds] = settings.autoplayCountdownSeconds this[SettingsTable.skipPlaylistAutoplayScreen] = settings.skipPlaylistAutoplayScreen this[SettingsTable.volume] = settings.volume this[SettingsTable.muted] = settings.muted @@ -105,6 +107,7 @@ internal fun SettingsItem.normalized(): SettingsItem = copy( defaultPlaybackSpeed = defaultPlaybackSpeed.takeIf { it.isFinite() } ?.coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED) ?: 1.0, accessMode = accessMode.toAccessMode(), + autoplayCountdownSeconds = autoplayCountdownSeconds.coerceIn(0, 60), sponsorBlockCategoryActions = sponsorBlockCategoryActions.withDefaultSponsorBlockCategoryActions(), sponsorBlockMinimumDuration = sponsorBlockMinimumDuration.coerceAtLeast(0), deArrowTitleMode = deArrowTitleMode.takeIf { it in DEARROW_TITLE_MODES } ?: "dearrow", diff --git a/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt index 9215d413..27fe8b0b 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsAutoplayOnOpenRoutesTest.kt @@ -49,6 +49,7 @@ class SettingsAutoplayOnOpenRoutesTest { val body = response.bodyAsText() assertTrue(body.contains("\"autoplay\":true")) assertTrue(body.contains("\"autoplayOnOpen\":true")) + assertTrue(body.contains("\"autoplayCountdownSeconds\":10")) } @Test @@ -69,4 +70,22 @@ class SettingsAutoplayOnOpenRoutesTest { assertTrue(body.contains("\"autoplay\":false")) assertTrue(body.contains("\"autoplayOnOpen\":true")) } + + @Test + fun `autoplay countdown is persisted with safe bounds`() = testApplication { + application { + install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) } + routing { settingsRoutes(service, auth) } + } + val response = client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"autoplayCountdownSeconds":120}""") + } + val body = client.get("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + }.bodyAsText() + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"autoplayCountdownSeconds\":60")) + } } From 3c14a3a1fb9908fec7f6ef346a7c3ed105ab6ed4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 14:53:44 +0000 Subject: [PATCH 048/137] feat: configure optional BiliBili cookies --- .env.example | 4 ++ .../downloader/BilibiliCookieContext.kt | 18 +++++++ .../server/downloader/OkHttpDownloader.kt | 1 + .../server/services/BilibiliCookieConfig.kt | 51 +++++++++++++++++++ .../server/services/NewPipeInitializer.kt | 21 ++++++++ .../server/BilibiliCookieContextTest.kt | 27 ++++++++++ .../services/BilibiliCookieConfigTest.kt | 45 ++++++++++++++++ 7 files changed, 167 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt create mode 100644 src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt diff --git a/.env.example b/.env.example index 0c395a83..7fadb293 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ DRAGONFLY_URL=redis://dragonfly:6379 DOWNLOADER_SERVICE_URL=http://typetype-downloader:18093 YOUTUBE_SESSION_ENCRYPTION_KEY=replace-with-at-least-32-random-characters YOUTUBE_OUTBOUND_PROXY_URL= +# Optional BiliBili account cookie header. Keep the value out of git and logs. +# BILIBILI_COOKIE=SESSDATA=...; bili_jct=...; buvid3=... +# Alternatively provide the cookie through a mounted secret file: +# BILIBILI_COOKIE_FILE=/run/secrets/bilibili_cookie YOUTUBE_REMOTE_LOGIN_SERVICE_URL=http://typetype-token:8081 YOUTUBE_REMOTE_LOGIN_CALLBACK_BASE_URL=http://typetype-server:8080 YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN=replace-with-shared-internal-token diff --git a/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt b/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt new file mode 100644 index 00000000..967ebfc2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.downloader + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull + +internal object BilibiliCookieContext { + @Volatile private var cookieHeader: String? = null + + fun set(value: String?) { + cookieHeader = value?.trim()?.takeIf { it.isNotEmpty() } + } + + fun headerFor(url: String): String? { + val parsed = url.toHttpUrlOrNull() ?: return null + val host = parsed.host.lowercase() + val isBilibiliApi = host == "bilibili.com" || host.endsWith(".bilibili.com") + return cookieHeader?.takeIf { isBilibiliApi } + } +} diff --git a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt index 1e9072ca..3e55b5f2 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt +++ b/src/main/kotlin/dev/typetype/server/downloader/OkHttpDownloader.kt @@ -113,6 +113,7 @@ class OkHttpDownloader private constructor( request.headers().forEach { (name, values) -> values.forEach { value -> builder.addHeader(name, value) } } + BilibiliCookieContext.headerFor(normalizedUrl)?.let { builder.header("Cookie", it) } YoutubeAuthUserContext.headerFor(normalizedUrl)?.let { builder.header(YOUTUBE_AUTH_USER_HEADER, it) } diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt new file mode 100644 index 00000000..6205538c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt @@ -0,0 +1,51 @@ +package dev.typetype.server.services + +/** Validates the optional instance-wide BiliBili cookie used by extraction. */ +data class BilibiliCookieConfig internal constructor( + val cookieHeader: String?, + val wasProvided: Boolean, +) { + val isConfigured: Boolean + get() = cookieHeader != null + + companion object { + private const val MAX_COOKIE_LENGTH = 32 * 1024 + private val COOKIE_NAME = Regex("[!#\\$%&'*+.^_`|~0-9A-Za-z-]+") + private val ACCOUNT_COOKIE_NAMES = setOf("SESSDATA", "bili_jct", "buvid3", "DedeUserID") + + fun fromEnvironment(read: (String) -> String? = System::getenv): BilibiliCookieConfig = + fromRaw(SecretConfigReader.read("BILIBILI_COOKIE", read)) + + internal fun fromRaw(raw: String?): BilibiliCookieConfig { + val supplied = raw?.trim()?.takeIf { it.isNotEmpty() } + ?: return BilibiliCookieConfig(cookieHeader = null, wasProvided = false) + return BilibiliCookieConfig(cookieHeader = normalize(supplied), wasProvided = true) + } + + private fun normalize(raw: String): String? { + if (raw.length > MAX_COOKIE_LENGTH || raw.any { it == '\r' || it == '\n' }) return null + val withoutHeader = if (raw.startsWith("Cookie:", ignoreCase = true)) { + raw.substringAfter(':').trim() + } else { + raw + } + if (withoutHeader.isEmpty()) return null + val names = HashSet() + val pairs = withoutHeader.split(';') + .map(String::trim) + .filter(String::isNotEmpty) + .map { pair -> + val separator = pair.indexOf('=') + if (separator <= 0) return null + val name = pair.substring(0, separator).trim() + val value = pair.substring(separator + 1).trim() + if (!COOKIE_NAME.matches(name) || value.isEmpty() || + value.any { it == '\r' || it == '\n' } || !names.add(name) + ) return null + name to value + } + if (pairs.isEmpty() || pairs.none { it.first in ACCOUNT_COOKIE_NAMES }) return null + return pairs.joinToString("; ") { (name, value) -> "$name=$value" } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index b6cc04b7..2d9493fb 100644 --- a/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -1,18 +1,25 @@ package dev.typetype.server.services +import dev.typetype.server.downloader.BilibiliCookieContext import dev.typetype.server.downloader.OkHttpDownloader import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder +import org.slf4j.LoggerFactory import java.net.ProxySelector object NewPipeInitializer { @Volatile private var initialized = false @Volatile private var decoderServiceUrl: String? = null + private val log = LoggerFactory.getLogger(NewPipeInitializer::class.java) + @Synchronized fun init( tokenServiceUrl: String? = null, youtubeProxySelector: ProxySelector? = null, + bilibiliCookie: BilibiliCookieConfig = BilibiliCookieConfig.fromEnvironment(), ): Unit { + configureBilibili(bilibiliCookie) NewPipe.setYoutubePlayerClient(YOUTUBE_PLAYER_CLIENT) val normalizedUrl = tokenServiceUrl?.trim()?.takeIf { it.isNotBlank() } if (normalizedUrl != null && normalizedUrl != decoderServiceUrl) { @@ -31,5 +38,19 @@ object NewPipeInitializer { NewPipe.setYoutubeSessionPoTokenProvider(TypetypeYoutubeSessionPoTokenProvider) } + private fun configureBilibili(config: BilibiliCookieConfig) { + ServiceList.BiliBili.setTokens(config.cookieHeader.orEmpty()) + ServiceList.BiliBili.setCookieFunctions( + if (config.isConfigured) BILIBILI_COOKIE_FUNCTIONS else emptySet(), + ) + BilibiliCookieContext.set(config.cookieHeader) + if (config.wasProvided && !config.isConfigured) { + log.warn("Ignoring invalid BILIBILI_COOKIE; expected a cookie header containing an account cookie") + } else if (config.isConfigured) { + log.info("BiliBili account cookie configured for instance extraction") + } + } + private const val YOUTUBE_PLAYER_CLIENT = "mweb" + private val BILIBILI_COOKIE_FUNCTIONS = setOf("high_res", "ai_subtitle") } diff --git a/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt new file mode 100644 index 00000000..17e0ff58 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt @@ -0,0 +1,27 @@ +package dev.typetype.server + +import dev.typetype.server.downloader.BilibiliCookieContext +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class BilibiliCookieContextTest { + @Test + fun `limits cookie injection to bilibili api hosts`() { + BilibiliCookieContext.set("SESSDATA=session; bili_jct=csrf") + try { + assertEquals( + "SESSDATA=session; bili_jct=csrf", + BilibiliCookieContext.headerFor("https://api.bilibili.com/x/player"), + ) + assertEquals( + "SESSDATA=session; bili_jct=csrf", + BilibiliCookieContext.headerFor("https://www.bilibili.com/video/BV1"), + ) + assertNull(BilibiliCookieContext.headerFor("https://upos-sz-mirrorali.bilivideo.com/video")) + assertNull(BilibiliCookieContext.headerFor("https://example.com/video")) + } finally { + BilibiliCookieContext.set(null) + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt b/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt new file mode 100644 index 00000000..11a13336 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.nio.file.Files + +class BilibiliCookieConfigTest { + @Test + fun `normalizes a browser cookie header`() { + val config = BilibiliCookieConfig.fromRaw( + "Cookie: SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", + ) + + assertTrue(config.wasProvided) + assertTrue(config.isConfigured) + assertEquals( + "SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", + config.cookieHeader, + ) + } + + @Test + fun `reads a mounted cookie file through the secret convention`() { + val file = Files.createTempFile("typetype-bilibili", ".cookie") + try { + Files.writeString(file, "SESSDATA=from-file; bili_jct=csrf") + val config = BilibiliCookieConfig.fromEnvironment { name -> + if (name == "BILIBILI_COOKIE_FILE") file.toString() else null + } + assertEquals("SESSDATA=from-file; bili_jct=csrf", config.cookieHeader) + } finally { + Files.deleteIfExists(file) + } + } + + @Test + fun `rejects malformed and unrelated cookies`() { + assertFalse(BilibiliCookieConfig.fromRaw("theme=dark").isConfigured) + assertNull(BilibiliCookieConfig.fromRaw("SESSDATA=session; SESSDATA=duplicate").cookieHeader) + assertNull(BilibiliCookieConfig.fromRaw("SESSDATA=session\r\nInjected: yes").cookieHeader) + } +} From 37e218a5a2dcb6166788a34ddb0083479ff3a820 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 14:58:26 +0000 Subject: [PATCH 049/137] fix: require HTTPS for BiliBili cookies --- .../dev/typetype/server/downloader/BilibiliCookieContext.kt | 2 +- .../kotlin/dev/typetype/server/BilibiliCookieContextTest.kt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt b/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt index 967ebfc2..e1750393 100644 --- a/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt +++ b/src/main/kotlin/dev/typetype/server/downloader/BilibiliCookieContext.kt @@ -13,6 +13,6 @@ internal object BilibiliCookieContext { val parsed = url.toHttpUrlOrNull() ?: return null val host = parsed.host.lowercase() val isBilibiliApi = host == "bilibili.com" || host.endsWith(".bilibili.com") - return cookieHeader?.takeIf { isBilibiliApi } + return cookieHeader?.takeIf { parsed.isHttps && isBilibiliApi } } } diff --git a/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt index 17e0ff58..a1805f74 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliCookieContextTest.kt @@ -18,6 +18,7 @@ class BilibiliCookieContextTest { "SESSDATA=session; bili_jct=csrf", BilibiliCookieContext.headerFor("https://www.bilibili.com/video/BV1"), ) + assertNull(BilibiliCookieContext.headerFor("http://api.bilibili.com/x/player")) assertNull(BilibiliCookieContext.headerFor("https://upos-sz-mirrorali.bilivideo.com/video")) assertNull(BilibiliCookieContext.headerFor("https://example.com/video")) } finally { From 2227216d79ab9ee9b9a42b6e30f9a474d057506c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 15:46:17 +0000 Subject: [PATCH 050/137] feat: add BiliBili session lifecycle with QR login --- .../dev/typetype/server/ServiceRegistry.kt | 7 ++ .../dev/typetype/server/db/DatabaseFactory.kt | 2 + .../server/db/tables/BiliBiliSessionsTable.kt | 14 +++ .../server/models/BiliBiliSessionModels.kt | 29 +++++ .../server/routes/BiliBiliSessionRoutes.kt | 54 ++++++++++ .../typetype/server/routes/UserDataRoutes.kt | 1 + .../server/services/BiliBiliQrLoginService.kt | 100 ++++++++++++++++++ .../server/services/BiliBiliSessionCrypto.kt | 52 +++++++++ .../server/services/BiliBiliSessionScope.kt | 60 +++++++++++ .../server/services/BiliBiliSessionService.kt | 77 ++++++++++++++ .../server/services/BiliBiliSessionStatus.kt | 12 +++ .../server/services/BiliBiliSessionStore.kt | 88 +++++++++++++++ .../services/BiliBiliSessionCryptoTest.kt | 36 +++++++ .../services/BiliBiliSessionScopeTest.kt | 28 +++++ 14 files changed, 560 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt create mode 100644 src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 5ca5063c..da983309 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -8,6 +8,8 @@ import dev.typetype.server.services.AllowedChannelsService import dev.typetype.server.services.AllowedPlaylistsService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService +import dev.typetype.server.services.BiliBiliSessionCrypto +import dev.typetype.server.services.BiliBiliSessionService import dev.typetype.server.services.BlockedService import dev.typetype.server.services.CustomAvatarService import dev.typetype.server.services.DeArrowService @@ -64,6 +66,11 @@ internal class ServiceRegistry( youtubeProxySelector, ) val youtubeSessionService = extraction.youtubeSessionService + val bilibiliSessionService = BiliBiliSessionService( + System.getenv("BILIBILI_SESSION_ENCRYPTION_KEY")?.let { secret -> + if (secret.length >= 32) BiliBiliSessionCrypto.fromSecret(secret) else null + }, + ) val authenticatedSabrInfoService = extraction.authenticatedSabrInfoService val youtubeSessionStreamService = extraction.youtubeSessionStreamService val youtubeSessionSabrStreamService = extraction.youtubeSessionSabrStreamService diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index 97e1c770..d24371b9 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -48,6 +48,7 @@ import dev.typetype.server.db.tables.UserTopicInterestTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable +import dev.typetype.server.db.tables.BiliBiliSessionsTable import dev.typetype.server.db.tables.YoutubeSessionsTable import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -100,6 +101,7 @@ object DatabaseFactory { YoutubeTakeoutPlaylistKeysTable, YoutubeSessionsTable, YoutubeSessionPairingsTable, + BiliBiliSessionsTable, BugReportsTable, NotificationStatesTable, NotificationReadItemsTable, diff --git a/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt new file mode 100644 index 00000000..0d9b51bc --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/BiliBiliSessionsTable.kt @@ -0,0 +1,14 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object BiliBiliSessionsTable : Table("bilibili_sessions") { + val userId = text("user_id") + val encryptedCookies = text("encrypted_cookies") + val status = text("status") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + val lastUsedAt = long("last_used_at").default(0) + val expiresAt = long("expires_at").default(0) + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt b/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt new file mode 100644 index 00000000..dfd9a6be --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class BiliBiliSessionStatusResponse( + val status: String, + val updatedAt: Long, + val lastUsedAt: Long, + val expiresAt: Long = 0, +) + +@Serializable +data class BiliBiliQrLoginResponse( + val qrUrl: String, + val qrcodeKey: String, + val expiresAt: Long, +) + +@Serializable +data class BiliBiliQrPollRequest( + val qrcodeKey: String, +) + +@Serializable +data class BiliBiliQrPollResponse( + val status: String, + val message: String = "", +) diff --git a/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt new file mode 100644 index 00000000..97451d4b --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt @@ -0,0 +1,54 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.BiliBiliQrPollRequest +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BiliBiliQrLoginResult +import dev.typetype.server.services.BiliBiliSessionService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +fun Route.biliBiliSessionRoutes(service: BiliBiliSessionService, authService: AuthService): Unit { + post("/bilibili-session/qr") { + call.withJwtAuth(authService) { _ -> + when (val result = service.startQrLogin()) { + is BiliBiliQrLoginResult.Success -> call.respond(HttpStatusCode.Created, result.response) + is BiliBiliQrLoginResult.Unavailable -> call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("BiliBili Session is unavailable", "bilibili_session_unavailable"), + ) + is BiliBiliQrLoginResult.Error -> call.respond( + HttpStatusCode.BadGateway, + ErrorResponse(result.message, "bilibili_qr_error"), + ) + } + } + } + post("/bilibili-session/qr/poll") { + call.withJwtAuth(authService) { userId -> + val request = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (request.qrcodeKey.isBlank()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing qrcodeKey")) + } + call.respond(service.pollQrLogin(userId, request.qrcodeKey)) + } + } + get("/bilibili-session/status") { + call.withJwtAuth(authService) { userId -> + call.respond(service.status(userId)) + } + } + delete("/bilibili-session") { + call.withJwtAuth(authService) { userId -> + service.delete(userId) + call.respond(HttpStatusCode.NoContent) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index e5c98696..600edf47 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -47,6 +47,7 @@ internal fun Route.userDataRoutes( notificationsRoutes(svc.notificationsService, authService) pushNotificationRoutes(svc.pushNotificationService, authService) youtubeSessionRoutes(svc.youtubeSessionService, authService) + biliBiliSessionRoutes(svc.bilibiliSessionService, authService) youtubeTakeoutImportRoutes(svc.youtubeTakeoutImportService, authService) profileRoutes(profileService, avatarService, svc.customAvatarService, authService) customAvatarRoutes(svc.customAvatarService, authService) diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt new file mode 100644 index 00000000..f1f24bdd --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt @@ -0,0 +1,100 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import okhttp3.OkHttpClient +import okhttp3.Request + +class BiliBiliQrLoginService( + private val client: OkHttpClient = OkHttpClient(), + private val nowMillis: () -> Long = System::currentTimeMillis, +) { + private val json = Json { ignoreUnknownKeys = true } + + suspend fun generate(): BiliBiliQrGenerateResult = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(GENERATE_URL) + .header("User-Agent", WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + val response = client.newCall(request).execute() + response.use { + val body = it.body.string() ?: return@withContext BiliBiliQrGenerateResult.Error("Empty response") + val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() + ?: return@withContext BiliBiliQrGenerateResult.Error("Invalid JSON") + val data = root["data"]?.jsonObject + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing data") + val url = data["url"]?.jsonPrimitive?.content + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing QR URL") + val key = data["qrcode_key"]?.jsonPrimitive?.content + ?: return@withContext BiliBiliQrGenerateResult.Error("Missing QR key") + BiliBiliQrGenerateResult.Success( + qrUrl = url, + qrcodeKey = key, + expiresAt = nowMillis() + QR_TTL_MS, + ) + } + } + + suspend fun poll(qrcodeKey: String): BiliBiliQrPollResult = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url("$POLL_URL?qrcode_key=$qrcodeKey") + .header("User-Agent", WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + val response = client.newCall(request).execute() + response.use { httpResponse -> + val body = httpResponse.body?.string() ?: return@withContext BiliBiliQrPollResult.Error("Empty response") + val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() + ?: return@withContext BiliBiliQrPollResult.Error("Invalid JSON") + val data = root["data"]?.jsonObject + ?: return@withContext BiliBiliQrPollResult.Error("Missing data") + val code = data["code"]?.jsonPrimitive?.content?.toIntOrNull() + ?: return@withContext BiliBiliQrPollResult.Error("Missing code") + when (code) { + 0 -> { + val cookies = httpResponse.headers("Set-Cookie") + .filter { it.contains("SESSDATA=") || it.contains("bili_jct=") || it.contains("buvid3=") } + .mapNotNull { cookie -> + val parts = cookie.split(";").firstOrNull()?.trim() + parts?.takeIf { it.isNotEmpty() } + } + val cookieHeader = cookies.joinToString("; ") + if (cookieHeader.contains("SESSDATA=")) { + BiliBiliQrPollResult.Confirmed(cookieHeader) + } else { + BiliBiliQrPollResult.Error("Missing SESSDATA in response") + } + } + 86038 -> BiliBiliQrPollResult.Expired + 86090 -> BiliBiliQrPollResult.Scanned + 86001 -> BiliBiliQrPollResult.Waiting + else -> BiliBiliQrPollResult.Error("Unexpected code: $code") + } + } + } + + companion object { + private const val GENERATE_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/generate" + private const val POLL_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/poll" + private const val QR_TTL_MS = 180_000L + private const val WEB_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + } +} + +sealed class BiliBiliQrGenerateResult { + data class Success(val qrUrl: String, val qrcodeKey: String, val expiresAt: Long) : BiliBiliQrGenerateResult() + data class Error(val message: String) : BiliBiliQrGenerateResult() +} + +sealed class BiliBiliQrPollResult { + data class Confirmed(val cookieHeader: String) : BiliBiliQrPollResult() + object Scanned : BiliBiliQrPollResult() + object Waiting : BiliBiliQrPollResult() + object Expired : BiliBiliQrPollResult() + data class Error(val message: String) : BiliBiliQrPollResult() +} diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt new file mode 100644 index 00000000..816735cf --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionCrypto.kt @@ -0,0 +1,52 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 +import javax.crypto.Cipher +import javax.crypto.spec.GCMParameterSpec +import javax.crypto.spec.SecretKeySpec + +class BiliBiliSessionCrypto private constructor(private val key: SecretKeySpec) { + private val random = SecureRandom() + + fun encrypt(value: String): String { + val nonce = ByteArray(NONCE_BYTES) + random.nextBytes(nonce) + val cipher = Cipher.getInstance(TRANSFORMATION) + cipher.init(Cipher.ENCRYPT_MODE, key, GCMParameterSpec(TAG_BITS, nonce)) + cipher.updateAAD(AAD) + val encrypted = cipher.doFinal(value.toByteArray(Charsets.UTF_8)) + return PREFIX + encoder.encodeToString(nonce + encrypted) + } + + fun decrypt(value: String): String { + require(value.startsWith(PREFIX)) { "Invalid encrypted payload format" } + val bytes = decoder.decode(value.removePrefix(PREFIX)) + require(bytes.size > NONCE_BYTES) { "Invalid encrypted payload" } + val nonce = bytes.copyOfRange(0, NONCE_BYTES) + val encrypted = bytes.copyOfRange(NONCE_BYTES, bytes.size) + val cipher = Cipher.getInstance(TRANSFORMATION) + cipher.init(Cipher.DECRYPT_MODE, key, GCMParameterSpec(TAG_BITS, nonce)) + cipher.updateAAD(AAD) + return cipher.doFinal(encrypted).toString(Charsets.UTF_8) + } + + companion object { + private const val PREFIX = "gcm256." + private const val NONCE_BYTES = 12 + private const val TAG_BITS = 128 + private const val MIN_SECRET_LENGTH = 32 + private const val TRANSFORMATION = "AES/GCM/NoPadding" + private val AAD = "typetype.bilibili-session".toByteArray(Charsets.UTF_8) + private val encoder = Base64.getUrlEncoder().withoutPadding() + private val decoder = Base64.getUrlDecoder() + + fun fromSecret(secret: String): BiliBiliSessionCrypto { + require(secret.length >= MIN_SECRET_LENGTH) { "BiliBili session encryption key is too short" } + val digest = MessageDigest.getInstance("SHA-256") + .digest(secret.toByteArray(Charsets.UTF_8)) + return BiliBiliSessionCrypto(SecretKeySpec(digest, "AES")) + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt new file mode 100644 index 00000000..c6c4c531 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt @@ -0,0 +1,60 @@ +package dev.typetype.server.services + +import dev.typetype.server.downloader.BilibiliCookieContext +import org.schabi.newpipe.extractor.ServiceList +import java.util.concurrent.Semaphore +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.util.concurrent.atomic.AtomicBoolean + +object BiliBiliSessionScope { + private const val PUBLIC_PERMITS = 64 + private val permits = Semaphore(PUBLIC_PERMITS, true) + + suspend fun withCredentials(cookieHeader: String, block: suspend () -> T): T = + withPermits(PUBLIC_PERMITS) { + val bilibili = ServiceList.BiliBili + try { + bilibili.setTokens(cookieHeader) + bilibili.setCookieFunctions(BILIBILI_COOKIE_FUNCTIONS) + BilibiliCookieContext.set(cookieHeader) + block() + } finally { + bilibili.setTokens("") + bilibili.setCookieFunctions(emptySet()) + BilibiliCookieContext.set(null) + } + } + + suspend fun withoutCredentials(block: suspend () -> T): T = + withPermits(1) { + val bilibili = ServiceList.BiliBili + bilibili.setTokens("") + bilibili.setCookieFunctions(emptySet()) + BilibiliCookieContext.set(null) + try { + block() + } finally { + BilibiliCookieContext.set(null) + } + } + + private suspend fun withPermits(count: Int, block: suspend () -> T): T { + val acquired = AtomicBoolean(false) + return try { + withContext(Dispatchers.IO) { + if (!permits.tryAcquire(count, PERMIT_ACQUIRE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + error("Timed out waiting for BiliBili extraction permits") + } + acquired.set(true) + } + block() + } finally { + if (acquired.get()) permits.release(count) + } + } + + private const val PERMIT_ACQUIRE_TIMEOUT_MS = 15_000L + private val BILIBILI_COOKIE_FUNCTIONS = setOf("high_res", "ai_subtitle") +} diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt new file mode 100644 index 00000000..898d36c1 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt @@ -0,0 +1,77 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BiliBiliQrLoginResponse +import dev.typetype.server.models.BiliBiliQrPollResponse +import dev.typetype.server.models.BiliBiliSessionStatusResponse + +class BiliBiliSessionService( + private val crypto: BiliBiliSessionCrypto?, + private val store: BiliBiliSessionStore = BiliBiliSessionStore(), + private val qrLoginService: BiliBiliQrLoginService = BiliBiliQrLoginService(), +) { + val isConfigured: Boolean = crypto != null + + suspend fun startQrLogin(): BiliBiliQrLoginResult { + if (!isConfigured) return BiliBiliQrLoginResult.Unavailable + return when (val result = qrLoginService.generate()) { + is BiliBiliQrGenerateResult.Success -> + BiliBiliQrLoginResult.Success( + BiliBiliQrLoginResponse( + qrUrl = result.qrUrl, + qrcodeKey = result.qrcodeKey, + expiresAt = result.expiresAt, + ), + ) + is BiliBiliQrGenerateResult.Error -> BiliBiliQrLoginResult.Error(result.message) + } + } + + suspend fun pollQrLogin(userId: String, qrcodeKey: String): BiliBiliQrPollResponse { + if (!isConfigured) return BiliBiliQrPollResponse("unavailable", "BiliBili session is unavailable") + val crypto = crypto ?: return BiliBiliQrPollResponse("unavailable", "BiliBili session is unavailable") + return when (val result = qrLoginService.poll(qrcodeKey)) { + is BiliBiliQrPollResult.Confirmed -> { + val cookies = BilibiliCookieConfig.fromRaw(result.cookieHeader) + if (!cookies.isConfigured) { + return BiliBiliQrPollResponse("error", "Invalid BiliBili cookies received") + } + store.completeForUser( + userId = userId, + encryptedCookies = crypto.encrypt(result.cookieHeader), + expiresAt = estimateExpiry(), + ) + BiliBiliQrPollResponse("confirmed") + } + is BiliBiliQrPollResult.Scanned -> BiliBiliQrPollResponse("scanned") + is BiliBiliQrPollResult.Waiting -> BiliBiliQrPollResponse("waiting") + is BiliBiliQrPollResult.Expired -> BiliBiliQrPollResponse("expired") + is BiliBiliQrPollResult.Error -> BiliBiliQrPollResponse("error", result.message) + } + } + + suspend fun status(userId: String): BiliBiliSessionStatusResponse = + if (isConfigured) store.status(userId) + else BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + + suspend fun delete(userId: String): Boolean = store.delete(userId) + + suspend fun connectedCookies(userId: String): String? { + val crypto = crypto ?: return null + val encrypted = store.connectedEncrypted(userId) ?: return null + val decrypted = runCatching { crypto.decrypt(encrypted) }.getOrNull() + if (decrypted == null) store.markNeedsReconnect(userId) + return decrypted + } + + private fun estimateExpiry(): Long = System.currentTimeMillis() + DEFAULT_COOKIE_TTL_MS + + companion object { + private const val DEFAULT_COOKIE_TTL_MS = 30L * 24 * 60 * 60 * 1000 + } +} + +sealed class BiliBiliQrLoginResult { + data class Success(val response: BiliBiliQrLoginResponse) : BiliBiliQrLoginResult() + data class Error(val message: String) : BiliBiliQrLoginResult() + object Unavailable : BiliBiliQrLoginResult() +} diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt new file mode 100644 index 00000000..04d5596a --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStatus.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.services + +enum class BiliBiliSessionStatus(val value: String) { + Connected("connected"), + NeedsReconnect("needs_reconnect"), + Disconnected("disconnected"); + + companion object { + fun from(value: String): BiliBiliSessionStatus = + entries.firstOrNull { it.value == value } ?: Disconnected + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt new file mode 100644 index 00000000..0e0bb325 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStore.kt @@ -0,0 +1,88 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.BiliBiliSessionsTable +import dev.typetype.server.models.BiliBiliSessionStatusResponse +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +class BiliBiliSessionStore( + private val nowMillis: () -> Long = System::currentTimeMillis, +) { + suspend fun completeForUser( + userId: String, + encryptedCookies: String, + expiresAt: Long, + ): Unit = DatabaseFactory.query { + upsertSession(userId, encryptedCookies, expiresAt, nowMillis()) + } + + suspend fun status(userId: String): BiliBiliSessionStatusResponse = DatabaseFactory.query { + BiliBiliSessionsTable.selectAll() + .where { BiliBiliSessionsTable.userId eq userId } + .singleOrNull() + ?.let { + BiliBiliSessionStatusResponse( + status = BiliBiliSessionStatus.from(it[BiliBiliSessionsTable.status]).value, + updatedAt = it[BiliBiliSessionsTable.updatedAt], + lastUsedAt = it[BiliBiliSessionsTable.lastUsedAt], + expiresAt = it[BiliBiliSessionsTable.expiresAt], + ) + } + ?: BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + } + + suspend fun delete(userId: String): Boolean = DatabaseFactory.query { + BiliBiliSessionsTable.deleteWhere { BiliBiliSessionsTable.userId eq userId } > 0 + } + + suspend fun connectedEncrypted(userId: String): String? = DatabaseFactory.query { + BiliBiliSessionsTable.selectAll() + .where { BiliBiliSessionsTable.userId eq userId } + .singleOrNull() + ?.takeIf { BiliBiliSessionStatus.from(it[BiliBiliSessionsTable.status]) == BiliBiliSessionStatus.Connected } + ?.get(BiliBiliSessionsTable.encryptedCookies) + } + + suspend fun markUsed(userId: String): Unit = DatabaseFactory.query { + BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { it[lastUsedAt] = nowMillis() } + } + + suspend fun markNeedsReconnect(userId: String): Unit = DatabaseFactory.query { + val now = nowMillis() + BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { + it[status] = BiliBiliSessionStatus.NeedsReconnect.value + it[updatedAt] = now + it[lastUsedAt] = now + } + } + + private fun upsertSession( + userId: String, + encryptedCookies: String, + expiresAt: Long, + now: Long, + ) { + val updated = BiliBiliSessionsTable.update({ BiliBiliSessionsTable.userId eq userId }) { + it[BiliBiliSessionsTable.encryptedCookies] = encryptedCookies + it[BiliBiliSessionsTable.status] = BiliBiliSessionStatus.Connected.value + it[BiliBiliSessionsTable.updatedAt] = now + it[BiliBiliSessionsTable.lastUsedAt] = 0 + it[BiliBiliSessionsTable.expiresAt] = expiresAt + } + if (updated == 0) { + BiliBiliSessionsTable.insert { + it[BiliBiliSessionsTable.userId] = userId + it[BiliBiliSessionsTable.encryptedCookies] = encryptedCookies + it[BiliBiliSessionsTable.status] = BiliBiliSessionStatus.Connected.value + it[BiliBiliSessionsTable.createdAt] = now + it[BiliBiliSessionsTable.updatedAt] = now + it[BiliBiliSessionsTable.lastUsedAt] = 0 + it[BiliBiliSessionsTable.expiresAt] = expiresAt + } + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt new file mode 100644 index 00000000..12cdb5a6 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionCryptoTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class BiliBiliSessionCryptoTest { + private val crypto = BiliBiliSessionCrypto.fromSecret("test-secret-key-at-least-32-chars!!") + + @Test + fun `encrypt and decrypt round-trips`() { + val cookie = "SESSDATA=abc123; bili_jct=csrf456; buvid3=device789" + val encrypted = crypto.encrypt(cookie) + assertNotEquals(cookie, encrypted) + assertEquals(cookie, crypto.decrypt(encrypted)) + } + + @Test + fun `encryption produces different ciphertexts`() { + val cookie = "SESSDATA=same" + assertNotEquals(crypto.encrypt(cookie), crypto.encrypt(cookie)) + } + + @Test + fun `decrypt rejects tampered payload`() { + val encrypted = crypto.encrypt("SESSDATA=test") + val tampered = encrypted.dropLast(4) + "AAAA" + assertThrows { crypto.decrypt(tampered) } + } + + @Test + fun `rejects short secret`() { + assertThrows { BiliBiliSessionCrypto.fromSecret("short") } + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt new file mode 100644 index 00000000..d03057f4 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt @@ -0,0 +1,28 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.ServiceList + +class BiliBiliSessionScopeTest { + @Test + fun `withCredentials sets and clears tokens`() = runBlocking { + val bilibili = ServiceList.BiliBili + BiliBiliSessionScope.withCredentials("SESSDATA=test; bili_jct=csrf") { + assertEquals("SESSDATA=test; bili_jct=csrf", bilibili.tokens) + } + assertEquals("", bilibili.tokens) + } + + @Test + fun `withoutCredentials clears tokens`() = runBlocking { + val bilibili = ServiceList.BiliBili + bilibili.setTokens("leftover") + BiliBiliSessionScope.withoutCredentials { + assertEquals("", bilibili.tokens) + } + assertEquals("", bilibili.tokens) + } +} From 89756d7ef83d068d4c09d4c291d19a76f8013bdc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 16:08:23 +0000 Subject: [PATCH 051/137] feat: add health check and SESSDATA expiry detection for BiliBili --- .../server/models/BiliBiliSessionModels.kt | 6 ++ .../server/routes/BiliBiliSessionRoutes.kt | 15 ++++ .../server/services/BiliBiliQrLoginService.kt | 2 +- .../server/services/BiliBiliSessionService.kt | 76 +++++++++++++++++-- 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt b/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt index dfd9a6be..675d710b 100644 --- a/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt +++ b/src/main/kotlin/dev/typetype/server/models/BiliBiliSessionModels.kt @@ -22,6 +22,12 @@ data class BiliBiliQrPollRequest( val qrcodeKey: String, ) +@Serializable +data class BiliBiliHealthResponse( + val status: String, + val message: String = "", +) + @Serializable data class BiliBiliQrPollResponse( val status: String, diff --git a/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt index 97451d4b..2c465755 100644 --- a/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/BiliBiliSessionRoutes.kt @@ -40,6 +40,21 @@ fun Route.biliBiliSessionRoutes(service: BiliBiliSessionService, authService: Au call.respond(service.pollQrLogin(userId, request.qrcodeKey)) } } + get("/bilibili-session/health") { + call.withJwtAuth(authService) { userId -> + val result = service.healthCheck(userId) + val status = when (result) { + is dev.typetype.server.services.BiliBiliHealthResult.Healthy -> "healthy" + is dev.typetype.server.services.BiliBiliHealthResult.Expired -> "expired" + is dev.typetype.server.services.BiliBiliHealthResult.RateLimited -> "rate_limited" + is dev.typetype.server.services.BiliBiliHealthResult.Disconnected -> "disconnected" + is dev.typetype.server.services.BiliBiliHealthResult.Unconfigured -> "unconfigured" + is dev.typetype.server.services.BiliBiliHealthResult.Error -> "error" + } + call.respond(dev.typetype.server.models.BiliBiliHealthResponse(status = status)) + } + } + get("/bilibili-session/status") { call.withJwtAuth(authService) { userId -> call.respond(service.status(userId)) diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt index f1f24bdd..23c130ba 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt @@ -81,7 +81,7 @@ class BiliBiliQrLoginService( private const val GENERATE_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/generate" private const val POLL_URL = "https://passport.bilibili.com/x/passport-login/web/qrcode/poll" private const val QR_TTL_MS = 180_000L - private const val WEB_USER_AGENT = + internal const val WEB_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" } } diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt index 898d36c1..d799e157 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt @@ -3,12 +3,23 @@ package dev.typetype.server.services import dev.typetype.server.models.BiliBiliQrLoginResponse import dev.typetype.server.models.BiliBiliQrPollResponse import dev.typetype.server.models.BiliBiliSessionStatusResponse +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import okhttp3.OkHttpClient +import okhttp3.Request class BiliBiliSessionService( private val crypto: BiliBiliSessionCrypto?, private val store: BiliBiliSessionStore = BiliBiliSessionStore(), private val qrLoginService: BiliBiliQrLoginService = BiliBiliQrLoginService(), + private val nowMillis: () -> Long = System::currentTimeMillis, ) { + private val json = Json { ignoreUnknownKeys = true } + private val client = OkHttpClient() + val isConfigured: Boolean = crypto != null suspend fun startQrLogin(): BiliBiliQrLoginResult { @@ -38,7 +49,7 @@ class BiliBiliSessionService( store.completeForUser( userId = userId, encryptedCookies = crypto.encrypt(result.cookieHeader), - expiresAt = estimateExpiry(), + expiresAt = parseSessDataExpiry(result.cookieHeader) ?: estimateExpiry(), ) BiliBiliQrPollResponse("confirmed") } @@ -49,9 +60,42 @@ class BiliBiliSessionService( } } - suspend fun status(userId: String): BiliBiliSessionStatusResponse = - if (isConfigured) store.status(userId) - else BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + suspend fun status(userId: String): BiliBiliSessionStatusResponse { + if (!isConfigured) return BiliBiliSessionStatusResponse(BiliBiliSessionStatus.Disconnected.value, 0, 0) + val stored = store.status(userId) + if (stored.status == BiliBiliSessionStatus.Connected.value && stored.expiresAt > 0 && stored.expiresAt < nowMillis()) { + store.markNeedsReconnect(userId) + return stored.copy(status = BiliBiliSessionStatus.NeedsReconnect.value) + } + return stored + } + + suspend fun healthCheck(userId: String): BiliBiliHealthResult { + if (!isConfigured) return BiliBiliHealthResult.Unconfigured + val cookies = connectedCookies(userId) ?: return BiliBiliHealthResult.Disconnected + return withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(NAV_API_URL) + .header("Cookie", cookies) + .header("User-Agent", BiliBiliQrLoginService.WEB_USER_AGENT) + .header("Referer", "https://www.bilibili.com/") + .build() + runCatching { + client.newCall(request).execute().use { response -> + if (response.code == 412) return@runCatching BiliBiliHealthResult.RateLimited + val body = response.body?.string() ?: return@runCatching BiliBiliHealthResult.Error("Empty response") + val root = json.parseToJsonElement(body).jsonObject + val code = root["code"]?.jsonPrimitive?.content?.toIntOrNull() ?: -1 + when (code) { + 0 -> BiliBiliHealthResult.Healthy + -101 -> BiliBiliHealthResult.Expired + -352 -> BiliBiliHealthResult.RateLimited + else -> BiliBiliHealthResult.Error("Unexpected code: $code") + } + } + }.getOrElse { BiliBiliHealthResult.Error(it.message ?: "Network error") } + } + } suspend fun delete(userId: String): Boolean = store.delete(userId) @@ -63,13 +107,35 @@ class BiliBiliSessionService( return decrypted } - private fun estimateExpiry(): Long = System.currentTimeMillis() + DEFAULT_COOKIE_TTL_MS + private fun parseSessDataExpiry(cookieHeader: String): Long? { + val sessData = cookieHeader.split(";") + .map { it.trim() } + .firstOrNull { it.startsWith("SESSDATA=", ignoreCase = true) } + ?.substringAfter("=") + ?: return null + val parts = sessData.split(",") + val timestamp = parts.getOrNull(1)?.trim()?.toLongOrNull() ?: return null + if (timestamp < 1_000_000_000L) return null + return timestamp * 1000 + } + + private fun estimateExpiry(): Long = nowMillis() + DEFAULT_COOKIE_TTL_MS companion object { private const val DEFAULT_COOKIE_TTL_MS = 30L * 24 * 60 * 60 * 1000 + private const val NAV_API_URL = "https://api.bilibili.com/x/web-interface/nav" } } +sealed class BiliBiliHealthResult { + object Healthy : BiliBiliHealthResult() + object Expired : BiliBiliHealthResult() + object RateLimited : BiliBiliHealthResult() + object Disconnected : BiliBiliHealthResult() + object Unconfigured : BiliBiliHealthResult() + data class Error(val message: String) : BiliBiliHealthResult() +} + sealed class BiliBiliQrLoginResult { data class Success(val response: BiliBiliQrLoginResponse) : BiliBiliQrLoginResult() data class Error(val message: String) : BiliBiliQrLoginResult() From 5bf16544f43abb515ea7d5e83e6f3bc01a81f680 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 16:38:06 +0000 Subject: [PATCH 052/137] feat: wire BiliBili session into stream extraction path --- .../server/ApplicationStreamRoutes.kt | 1 + .../dev/typetype/server/ServiceRegistry.kt | 5 ++++ .../server/routes/StreamRouteDependencies.kt | 1 + .../typetype/server/routes/StreamRoutes.kt | 10 +++++++ .../server/services/BiliBiliSessionService.kt | 4 +++ .../services/BiliBiliSessionStreamService.kt | 28 +++++++++++++++++++ 6 files changed, 49 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt diff --git a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 2e729658..34740596 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -33,6 +33,7 @@ internal fun Route.installStreamRoutes( publicHlsManifestTokenService = svc.publicHlsManifestTokenService, providerMediaHandleService = svc.providerMediaHandleService, sabrStreamContractFilter = { url, data -> data.withPlayableSabrStreams(url, svc.sabrSessionStore) }, + bilibiliSessionStreamInfo = { userId, url -> svc.bilibiliSessionStreamService.getStreamInfo(userId, url) }, youtubeSessionSabrStreamInfo = svc.youtubeSessionSabrStreamService?.let { service -> { userId, url -> service.getStreamInfo(userId, url) } }, diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index da983309..ed583493 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -10,6 +10,7 @@ import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AudioOnlyMediaTokenService import dev.typetype.server.services.BiliBiliSessionCrypto import dev.typetype.server.services.BiliBiliSessionService +import dev.typetype.server.services.BiliBiliSessionStreamService import dev.typetype.server.services.BlockedService import dev.typetype.server.services.CustomAvatarService import dev.typetype.server.services.DeArrowService @@ -78,6 +79,10 @@ internal class ServiceRegistry( val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService val bilibiliStreamService = extraction.bilibiliStreamService + val bilibiliSessionStreamService = BiliBiliSessionStreamService( + bilibiliStreamService, + bilibiliSessionService, + ) val streamService = extraction.streamService val searchService = extraction.searchService val trendingService = extraction.trendingService diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt index 24addc64..b4bde0f4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRouteDependencies.kt @@ -18,4 +18,5 @@ internal data class StreamRouteDependencies( val providerMediaHandleService: ProviderMediaHandleService?, val sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)?, val youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)?, + val bilibiliSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)?, ) diff --git a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index bd033d15..9fb17cb0 100644 --- a/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -41,6 +41,7 @@ fun Route.streamRoutes( bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, + bilibiliSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, ) { val dependencies = StreamRouteDependencies( @@ -52,6 +53,7 @@ fun Route.streamRoutes( providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrStreamContractFilter, youtubeSessionSabrStreamInfo = youtubeSessionSabrStreamInfo, + bilibiliSessionStreamInfo = bilibiliSessionStreamInfo, ) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) streamRoute( @@ -189,6 +191,14 @@ private suspend fun resolveStreamInfo( publicResult: ExtractionResult, dependencies: StreamRouteDependencies, ): StreamResolution { + if (deliveryMode == StreamDeliveryMode.BiliBili) { + val bilibiliInfo = dependencies.bilibiliSessionStreamInfo + if (bilibiliInfo != null && userId != null) { + val bilibiliResult = bilibiliInfo(userId, url) + if (bilibiliResult != null) return StreamResolution(bilibiliResult, authenticated = true) + } + return StreamResolution(publicResult) + } val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo if (!deliveryMode.isSabr() || authenticatedInfo == null) { return StreamResolution(publicResult) diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt index d799e157..93194b0c 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionService.kt @@ -107,6 +107,10 @@ class BiliBiliSessionService( return decrypted } + suspend fun markUsed(userId: String) = store.markUsed(userId) + + suspend fun markNeedsReconnect(userId: String) = store.markNeedsReconnect(userId) + private fun parseSessDataExpiry(cookieHeader: String): Long? { val sessData = cookieHeader.split(";") .map { it.trim() } diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt new file mode 100644 index 00000000..a6672724 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt @@ -0,0 +1,28 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class BiliBiliSessionStreamService( + private val streamService: StreamService, + private val sessionService: BiliBiliSessionService, +) { + suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { + if (!isBiliBiliUrl(url)) return null + val cookies = sessionService.connectedCookies(userId) ?: return null + return BiliBiliSessionScope.withCredentials(cookies) { + val result = streamService.getStreamInfo(url) + if (result is ExtractionResult.Success) sessionService.markUsed(userId) + if (result is ExtractionResult.Failure && requiresReconnect(result)) sessionService.markNeedsReconnect(userId) + result + } + } + + private fun isBiliBiliUrl(url: String): Boolean = + url.contains("bilibili.com") || url.contains("b23.tv") + + private fun requiresReconnect(result: ExtractionResult.Failure): Boolean = + result.message.contains("412") || + result.message.contains("-352") || + result.message.contains("risk", ignoreCase = true) +} From 46f2235916ea592dcb6e4f71c38c46fd6638eec0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 16:51:49 +0000 Subject: [PATCH 053/137] feat: add per-user rate limiting for BiliBili extraction --- .../server/services/BiliBiliSessionScope.kt | 25 +++++++++++++++++-- .../services/BiliBiliSessionStreamService.kt | 22 ++++++++++++---- .../services/BiliBiliSessionScopeTest.kt | 2 +- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt index c6c4c531..b2e8944f 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionScope.kt @@ -2,6 +2,7 @@ package dev.typetype.server.services import dev.typetype.server.downloader.BilibiliCookieContext import org.schabi.newpipe.extractor.ServiceList +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Semaphore import java.util.concurrent.TimeUnit import kotlinx.coroutines.Dispatchers @@ -11,9 +12,13 @@ import java.util.concurrent.atomic.AtomicBoolean object BiliBiliSessionScope { private const val PUBLIC_PERMITS = 64 private val permits = Semaphore(PUBLIC_PERMITS, true) + private val userRequestTimestamps = ConcurrentHashMap>() + private const val MAX_REQUESTS_PER_WINDOW = 30 + private const val RATE_WINDOW_MS = 60_000L - suspend fun withCredentials(cookieHeader: String, block: suspend () -> T): T = - withPermits(PUBLIC_PERMITS) { + suspend fun withCredentials(userId: String, cookieHeader: String, block: suspend () -> T): T { + checkRateLimit(userId) + return withPermits(PUBLIC_PERMITS) { val bilibili = ServiceList.BiliBili try { bilibili.setTokens(cookieHeader) @@ -26,6 +31,7 @@ object BiliBiliSessionScope { BilibiliCookieContext.set(null) } } + } suspend fun withoutCredentials(block: suspend () -> T): T = withPermits(1) { @@ -40,6 +46,18 @@ object BiliBiliSessionScope { } } + private fun checkRateLimit(userId: String) { + val now = System.currentTimeMillis() + val queue = userRequestTimestamps.computeIfAbsent(userId) { ArrayDeque() } + synchronized(queue) { + while (queue.isNotEmpty() && queue.first() < now - RATE_WINDOW_MS) queue.removeFirst() + if (queue.size >= MAX_REQUESTS_PER_WINDOW) { + throw BiliBiliRateLimitException(MAX_REQUESTS_PER_WINDOW) + } + queue.addLast(now) + } + } + private suspend fun withPermits(count: Int, block: suspend () -> T): T { val acquired = AtomicBoolean(false) return try { @@ -58,3 +76,6 @@ object BiliBiliSessionScope { private const val PERMIT_ACQUIRE_TIMEOUT_MS = 15_000L private val BILIBILI_COOKIE_FUNCTIONS = setOf("high_res", "ai_subtitle") } + +class BiliBiliRateLimitException(val maxRequests: Int) : + RuntimeException("BiliBili rate limit exceeded: max $maxRequests requests per minute") diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt index a6672724..4415b3bc 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliSessionStreamService.kt @@ -10,11 +10,18 @@ class BiliBiliSessionStreamService( suspend fun getStreamInfo(userId: String, url: String): ExtractionResult? { if (!isBiliBiliUrl(url)) return null val cookies = sessionService.connectedCookies(userId) ?: return null - return BiliBiliSessionScope.withCredentials(cookies) { - val result = streamService.getStreamInfo(url) - if (result is ExtractionResult.Success) sessionService.markUsed(userId) - if (result is ExtractionResult.Failure && requiresReconnect(result)) sessionService.markNeedsReconnect(userId) - result + return try { + BiliBiliSessionScope.withCredentials(userId, cookies) { + val result = streamService.getStreamInfo(url) + if (result is ExtractionResult.Success) sessionService.markUsed(userId) + if (result is ExtractionResult.Failure && requiresReconnect(result)) sessionService.markNeedsReconnect(userId) + result + } + } catch (e: BiliBiliRateLimitException) { + ExtractionResult.Failure( + BILIBILI_RATE_LIMIT_MESSAGE, + BILIBILI_RATE_LIMIT_CODE, + ) } } @@ -25,4 +32,9 @@ class BiliBiliSessionStreamService( result.message.contains("412") || result.message.contains("-352") || result.message.contains("risk", ignoreCase = true) + + companion object { + const val BILIBILI_RATE_LIMIT_MESSAGE = "BiliBili is temporarily limiting requests. Try again later." + const val BILIBILI_RATE_LIMIT_CODE = "bilibili_rate_limited" + } } diff --git a/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt index d03057f4..34d37602 100644 --- a/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/BiliBiliSessionScopeTest.kt @@ -10,7 +10,7 @@ class BiliBiliSessionScopeTest { @Test fun `withCredentials sets and clears tokens`() = runBlocking { val bilibili = ServiceList.BiliBili - BiliBiliSessionScope.withCredentials("SESSDATA=test; bili_jct=csrf") { + BiliBiliSessionScope.withCredentials("test-user", "SESSDATA=test; bili_jct=csrf") { assertEquals("SESSDATA=test; bili_jct=csrf", bilibili.tokens) } assertEquals("", bilibili.tokens) From a32f2c75d00f1ba9caad60be5e3832b7376b658b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 20:14:03 +0000 Subject: [PATCH 054/137] fix: keep BiliBili QR polling alive --- .../server/services/BiliBiliQrLoginService.kt | 13 +++++++++---- .../services/BiliBiliQrLoginServiceTest.kt | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt diff --git a/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt index 23c130ba..9d64292e 100644 --- a/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt +++ b/src/main/kotlin/dev/typetype/server/services/BiliBiliQrLoginService.kt @@ -69,10 +69,7 @@ class BiliBiliQrLoginService( BiliBiliQrPollResult.Error("Missing SESSDATA in response") } } - 86038 -> BiliBiliQrPollResult.Expired - 86090 -> BiliBiliQrPollResult.Scanned - 86001 -> BiliBiliQrPollResult.Waiting - else -> BiliBiliQrPollResult.Error("Unexpected code: $code") + else -> classifyBiliBiliQrCode(code) } } } @@ -86,6 +83,14 @@ class BiliBiliQrLoginService( } } +internal fun classifyBiliBiliQrCode(code: Int): BiliBiliQrPollResult = when (code) { + 86038 -> BiliBiliQrPollResult.Expired + 86090 -> BiliBiliQrPollResult.Scanned + // The current web endpoint uses 86101 while the QR is still waiting to be scanned. + 86001, 86101 -> BiliBiliQrPollResult.Waiting + else -> BiliBiliQrPollResult.Error("Unexpected code: $code") +} + sealed class BiliBiliQrGenerateResult { data class Success(val qrUrl: String, val qrcodeKey: String, val expiresAt: Long) : BiliBiliQrGenerateResult() data class Error(val message: String) : BiliBiliQrGenerateResult() diff --git a/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt b/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt new file mode 100644 index 00000000..2f2a2cee --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/services/BiliBiliQrLoginServiceTest.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class BiliBiliQrLoginServiceTest { + @Test + fun code_86101_is_waiting() { + assertTrue(classifyBiliBiliQrCode(86101) is BiliBiliQrPollResult.Waiting) + } + + @Test + fun code_86090_is_scanned() { + assertTrue(classifyBiliBiliQrCode(86090) is BiliBiliQrPollResult.Scanned) + } +} From ca3daa03ab059b861c10e3cb4142c748b739cbdf Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 20 Sep 2026 20:51:25 +0000 Subject: [PATCH 055/137] chore: bump server to 1.8.2 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3e32a72b..5cc6396e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.8.1 +appVersion=1.8.2 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From e8aad979508c5df46cca1b9fbd530d62de5d933f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 08:49:10 +0000 Subject: [PATCH 056/137] fix: copy Gradle modules in image build --- Dockerfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index db8e4f19..d4e986b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,22 @@ COPY gradlew ./ COPY gradle/ ./gradle/ RUN ./gradlew --version --no-daemon -q COPY build.gradle.kts gradle.properties settings.gradle.kts* ./ -RUN ./gradlew dependencies --no-daemon -q || true COPY src/ ./src/ +COPY server-admin/ ./server-admin/ +COPY server-auth/ ./server-auth/ +COPY server-cache/ ./server-cache/ +COPY server-core/ ./server-core/ +COPY server-db/ ./server-db/ +COPY server-domain/ ./server-domain/ +COPY server-downloader/ ./server-downloader/ +COPY server-http/ ./server-http/ +COPY server-playback/ ./server-playback/ +COPY server-portability/ ./server-portability/ +COPY server-sabr/ ./server-sabr/ +COPY server-test-support/ ./server-test-support/ +COPY server-services/ ./server-services/ +COPY server-token-gateway/ ./server-token-gateway/ +RUN ./gradlew dependencies --no-daemon -q || true RUN ./gradlew shadowJar --no-daemon -q -PappVersion="$APP_VERSION" FROM eclipse-temurin:25-jre-alpine AS runner From 265837a2956ddb78f76c26ce79fe0c9486dc14d7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 09:56:22 +0000 Subject: [PATCH 057/137] chore: update PipePipeExtractor dependency --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index eeffcaa7..629b8171 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,7 +57,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") From 0249ea227f254b93a7be97612e72a3d5820714a1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 10:19:08 +0000 Subject: [PATCH 058/137] ci: build affected server modules --- .github/scripts/ci-affected-tasks.sh | 165 +++++++++++++++++++++++++++ .github/workflows/ci.yml | 20 +++- 2 files changed, 183 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/ci-affected-tasks.sh diff --git a/.github/scripts/ci-affected-tasks.sh b/.github/scripts/ci-affected-tasks.sh new file mode 100755 index 00000000..ab913acb --- /dev/null +++ b/.github/scripts/ci-affected-tasks.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +event_name="${GITHUB_EVENT_NAME:-push}" +base_ref="${GITHUB_BASE_REF:-}" +before="${GITHUB_EVENT_BEFORE:-}" +zero_sha="0000000000000000000000000000000000000000" +sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" +output_file="${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +emit_tasks() { + if [[ -z "$1" ]]; then + return + fi + + { + echo "tasks<>"$output_file" +} + +if [[ "$event_name" != "push" && "$event_name" != "pull_request" ]]; then + emit_tasks "build" + exit 0 +fi + +if [[ "$event_name" == "pull_request" ]]; then + if [[ -z "$base_ref" ]]; then + echo "Missing base ref for pull request" >&2 + exit 1 + fi + + git fetch --no-tags origin "$base_ref" + base_commit="$(git rev-parse "origin/$base_ref")" + mapfile -d '' -t changed_files < <( + git diff --name-only -z "$base_commit...$sha" + ) +elif [[ -n "$before" && "$before" != "$zero_sha" ]] && + git cat-file -e "$before^{commit}" 2>/dev/null; then + mapfile -d '' -t changed_files < <( + git diff --name-only -z "$before...$sha" + ) +elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then + mapfile -d '' -t changed_files < <( + git diff --name-only -z "HEAD^...$sha" + ) +else + emit_tasks "build" + exit 0 +fi + +is_shared_change() { + case "$1" in + .github/scripts/* | .github/workflows/ci.yml | .github/workflows/coverage.yml) + return 0 + ;; + .github/workflows/openapi.yml) + return 0 + ;; + build.gradle.kts | settings.gradle.kts | gradle.properties | gradlew | gradle/*) + return 0 + ;; + openapi.yaml | openapi/* | src/*) + return 0 + ;; + Dockerfile) + return 0 + ;; + esac + + return 1 +} + +is_ignored_change() { + case "$1" in + *.md | *.png | *.svg | .gitignore | .editorconfig | LICENSE*) + return 0 + ;; + docs/* | .github/ISSUE_TEMPLATE/*) + return 0 + ;; + esac + + return 1 +} + +full_build=false +declare -A changed_modules=() + +for file in "${changed_files[@]}"; do + if is_ignored_change "$file"; then + continue + fi + + if is_shared_change "$file"; then + full_build=true + continue + fi + + if [[ "$file" == server-*/* ]]; then + module="${file%%/*}" + changed_modules["$module"]=1 + else + full_build=true + fi +done + +if [[ "$full_build" == true ]]; then + emit_tasks "build" + exit 0 +fi + +if [[ ${#changed_modules[@]} -eq 0 ]]; then + exit 0 +fi + +declare -A dependents_by_module=() +for build_file in server-*/build.gradle.kts; do + module="${build_file%/*}" + while IFS= read -r line; do + if [[ "$line" =~ project\(\":([A-Za-z0-9_-]+)\"\) ]]; then + dependency="${BASH_REMATCH[1]}" + dependents_by_module["$dependency"]+="$module " + fi + done <"$build_file" +done + +declare -A affected_modules=() + +mark_affected() { + local module="$1" + + if [[ -n "${affected_modules[$module]:-}" ]]; then + return + fi + + affected_modules["$module"]=1 + + local dependent + for dependent in ${dependents_by_module["$module"]:-}; do + mark_affected "$dependent" + done +} + +for module in "${!changed_modules[@]}"; do + if [[ ! -f "$module/build.gradle.kts" ]]; then + echo "Changed module no longer exists: $module" >&2 + emit_tasks "build" + exit 0 + fi + + mark_affected "$module" +done + +module_tasks="$( + for module in "${!affected_modules[@]}"; do + printf ':%s:build\n' "$module" + done | sort +)" + +emit_tasks "$module_tasks" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f5edc8f..ff9408df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: runs-on: [self-hosted, Linux, X64, ci-stack] steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" @@ -29,5 +31,19 @@ jobs: - uses: gradle/actions/setup-gradle@v6 - - name: Build - run: ./gradlew build --max-workers=8 + - name: Detect affected Gradle modules + id: affected + run: bash .github/scripts/ci-affected-tasks.sh + + - name: Build affected modules + env: + TASKS: ${{ steps.affected.outputs.tasks }} + run: | + if [[ -z "$TASKS" ]]; then + echo "No Gradle compilation or tests are required." + exit 0 + fi + + readarray -t task_args <<<"$TASKS" + printf 'Running:\n%s\n' "$TASKS" + ./gradlew "${task_args[@]}" --parallel --max-workers=32 From cffb8415fd87b1716c16b0eeb74f55a3dfe854b8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 14:13:37 +0000 Subject: [PATCH 059/137] fix: align SABR duration with media end --- .../dev/typetype/server/routes/SabrPlaybackWindowTiming.kt | 2 +- .../dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt index 9d0aee52..a13efd6a 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt @@ -12,7 +12,7 @@ internal fun SabrSessionHolder.durationMs(): Long { } val audioEndMs = indexedEndMs(audioFormat) val videoEndMs = indexedEndMs(videoFormat) - if (audioEndMs > 0L && videoEndMs > 0L) return maxOf(audioEndMs, videoEndMs) + if (audioEndMs > 0L && videoEndMs > 0L) return minOf(audioEndMs, videoEndMs) return maxOf(audioFormat.approxDurationMs, videoFormat.approxDurationMs, 0L) } diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt index 19ddf9d7..83b00110 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilderTest.kt @@ -232,7 +232,7 @@ class SabrPlaybackWindowBuilderTest { ) assertTrue(result.isReady) assertTrue(result.response.endOfStream) - assertEquals(898_000L, result.response.durationMs) + assertEquals(897_000L, result.response.durationMs) coVerify(exactly = 0) { store.cachedSegment(holder, match { it.sequenceNumber > 100 }) } } From 7c1fd0e8e7e427e7dd59f5858039c1071e3e65ee Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 16:57:43 +0000 Subject: [PATCH 060/137] test: cover live session eviction and cache bounds --- .../services/SabrMemorySegmentCacheTest.kt | 45 +++++++++++++++++++ .../server/services/SabrSessionStoreTest.kt | 15 +++++++ 2 files changed, 60 insertions(+) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt new file mode 100644 index 00000000..e78a6b11 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMemorySegmentCacheTest.kt @@ -0,0 +1,45 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class SabrMemorySegmentCacheTest { + @Test + fun `eviction behind reader tail keeps the boundary and newer media`() { + val cache = SabrMemorySegmentCache(maxBytes = 1_000) + cache.put("before-tail", segment(startMs = 0L, durationMs = 5L)) + cache.put("at-tail", segment(startMs = 5L, durationMs = 5L)) + cache.put("ahead", segment(startMs = 10L, durationMs = 5L)) + + cache.evictBefore(6L) + + assertNull(cache.get("before-tail")) + assertNotNull(cache.get("at-tail")) + assertNotNull(cache.get("ahead")) + } + + @Test + fun `least recently used media is removed when the byte limit is reached`() { + val cache = SabrMemorySegmentCache(maxBytes = 10L) + cache.put("oldest", segment(bytes = ByteArray(6))) + cache.put("newest", segment(bytes = ByteArray(6))) + + assertNull(cache.get("oldest")) + assertNotNull(cache.get("newest")) + } + + private fun segment( + startMs: Long = 0L, + durationMs: Long = 1L, + bytes: ByteArray = ByteArray(1), + ): CachedSabrSegment = CachedSabrSegment( + itag = 137, + sequence = 1, + init = false, + startMs = startMs, + durationMs = durationMs, + mimeType = "video/webm", + bytes = bytes, + ) +} diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt index 7bc34a9b..ba39f5b9 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt @@ -54,6 +54,21 @@ class SabrSessionRegistryTest { assertSame(holder, registry.lookupByToken("token-playback")) } + @Test + fun `active live playback requests refresh idle eviction timestamp`() { + val registry = SabrSessionRegistry() + val holder = holder("live", Instant.EPOCH) + holder.markExpectedLive() + registry.put(holder.key, holder) + + assertSame(holder, registry.lookupByToken("live", holder.sessionToken)) + assertSame(holder, registry.lookupByToken("live", holder.sessionToken, holder.audioFormat.itag)) + + registry.evictIdle(Instant.now().minusSeconds(1)) + + assertSame(holder, registry.lookupByToken(holder.sessionToken)) + } + @Test fun `healthy session remains reusable`() { val registry = SabrSessionRegistry() From a8cbd09f65e362b16468c399cf316ccf0148d74c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 17:41:19 +0000 Subject: [PATCH 061/137] test: cover playback lookup by format --- .../kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt index ba39f5b9..f1569bc7 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt @@ -63,6 +63,7 @@ class SabrSessionRegistryTest { assertSame(holder, registry.lookupByToken("live", holder.sessionToken)) assertSame(holder, registry.lookupByToken("live", holder.sessionToken, holder.audioFormat.itag)) + assertSame(holder, registry.lookupByItag("live", "user", holder.audioFormat.itag)) registry.evictIdle(Instant.now().minusSeconds(1)) From e8aeeb613693d603aba05e64bbfeec4afa837d13 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 17:41:19 +0000 Subject: [PATCH 062/137] fix: expose browser-compatible SABR audio --- .../server/routes/SabrManifestHandler.kt | 4 +- .../server/routes/SabrPlaybackHandler.kt | 4 +- .../routes/SabrSessionDescriptorHandler.kt | 4 +- .../server/routes/SabrStreamContractFilter.kt | 49 +++++++++++++++- .../routes/SabrStreamContractFilterTest.kt | 57 +++++++++++++++++++ 5 files changed, 109 insertions(+), 9 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt index 6f4fce76..e10ac666 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt @@ -89,7 +89,7 @@ internal class SabrManifestHandler( prepared.info, audioToken?.selectedItag ?: call.request.queryParameters["audioItag"]?.toIntOrNull(), audioToken?.selectedAudioTrackId ?: call.request.queryParameters["audioTrackId"], - requireAac = true, + requireAac = false, ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, @@ -149,7 +149,7 @@ internal class SabrManifestHandler( if (preflight(holder, startTimeMs)) return holder sabrSessionStore.release(holder) val refreshed = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = false) ?: prepared - val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = true) + val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = false) ?: return null val refreshedVideo = SabrFormatSelector.video(refreshed.info, video.itag) ?: return null val fresh = createHolder(videoId, userId, refreshed, refreshedAudio, refreshedVideo, startTimeMs, purpose) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 7f7f26dc..6c039d3d 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -72,7 +72,7 @@ internal class SabrPlaybackHandler( prepared.info, request.audioItag ?: holder.audioFormat.itag, request.audioTrackId ?: holder.audioFormat.audioTrackId, - requireAac = true, + requireAac = false, ) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) val video = SabrFormatSelector.video(prepared.info, request.videoItag ?: holder.videoFormat.itag) ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR video for this video")) @@ -184,7 +184,7 @@ internal class SabrPlaybackHandler( prepared.info, request.audioItag, request.audioTrackId, - requireAac = true, + requireAac = false, ) ?: run { call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video")) null diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt index 97b9c92a..d292c9ec 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrSessionDescriptorHandler.kt @@ -56,7 +56,7 @@ internal class SabrSessionDescriptorHandler( prepared.info, call.request.queryParameters["audioItag"]?.toIntOrNull(), call.request.queryParameters["audioTrackId"], - requireAac = true, + requireAac = false, ) ?: return call.respond( HttpStatusCode.UnprocessableEntity, ErrorResponse("No SABR audio for this video", "no_playable_streams"), @@ -109,7 +109,7 @@ internal class SabrSessionDescriptorHandler( if (preflight(holder, startTimeMs)) return holder sabrSessionStore.release(holder) val refreshed = sabrSessionStore.fetchInfo(videoId, startTimeMs, cachedFirst = false) ?: prepared - val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = true) + val refreshedAudio = SabrFormatSelector.audio(refreshed.info, audio.itag, audio.audioTrackId, requireAac = false) ?: return null val refreshedVideo = SabrFormatSelector.video(refreshed.info, video.itag) ?: return null val fresh = createHolder(videoId, userId, refreshed, refreshedAudio, refreshedVideo, startTimeMs) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt index f966d9a4..9871b7fb 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt @@ -6,6 +6,8 @@ import dev.typetype.server.models.VideoStreamItem import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo +import java.net.URLEncoder +import java.nio.charset.StandardCharsets internal suspend fun StreamResponse.withPlayableSabrStreams( url: String, @@ -15,13 +17,14 @@ internal suspend fun StreamResponse.withPlayableSabrStreams( val videoId = url.youtubeVideoId() ?: return this val prepared = sabrSessionStore.fetchInfo(videoId, cachedFirst = true) ?: return withoutSabrStreams() - val hasAudio = SabrFormatSelector.audio(prepared.info, null, null, requireAac = true) != null + val hasAudio = SabrFormatSelector.audio(prepared.info, null, null, requireAac = false) != null if (!hasAudio) return withoutSabrStreams() val enriched = withMissingSabrVideoStreams(videoId, prepared.info) + .withMissingSabrAudioStreams(videoId, prepared.info) return enriched.copy( videoStreams = enriched.videoStreams.filter { it.isPlayableSabrVideo(prepared.info) }, videoOnlyStreams = enriched.videoOnlyStreams.filter { it.isPlayableSabrVideo(prepared.info) }, - audioStreams = audioStreams.filter { it.isPlayableSabrAudio(prepared.info) }, + audioStreams = enriched.audioStreams.filter { it.isPlayableSabrAudio(prepared.info) }, ) } @@ -35,6 +38,17 @@ private fun StreamResponse.withMissingSabrVideoStreams(videoId: String, info: Yo return copy(videoOnlyStreams = videoOnlyStreams + missing) } +private fun StreamResponse.withMissingSabrAudioStreams(videoId: String, info: YoutubeSabrInfo): StreamResponse { + val existing = audioStreams.map { it.itag to it.audioTrackId }.toSet() + val missing = info.formats.asSequence() + .filter { it.isAudio && (it.itag to it.audioTrackId) !in existing } + .filter { SabrFormatSelector.audio(info, it.itag, it.audioTrackId, requireAac = false) != null } + .mapNotNull { it.toAudioStreamItem(videoId) } + .toList() + if (missing.isEmpty()) return this + return copy(audioStreams = audioStreams + missing) +} + internal fun StreamResponse.hasSabrStreams(): Boolean = videoStreams.any { it.deliveryMethod == SABR_DELIVERY_METHOD } || videoOnlyStreams.any { it.deliveryMethod == SABR_DELIVERY_METHOD } || @@ -59,7 +73,7 @@ private fun VideoStreamItem.isPlayableSabrVideo(info: YoutubeSabrInfo): Boolean private fun AudioStreamItem.isPlayableSabrAudio(info: YoutubeSabrInfo): Boolean = deliveryMethod != SABR_DELIVERY_METHOD || - SabrFormatSelector.audio(info, itag, audioTrackId, requireAac = true) != null + SabrFormatSelector.audio(info, itag, audioTrackId, requireAac = false) != null private fun YoutubeSabrFormat.toVideoStreamItem(videoId: String, info: YoutubeSabrInfo): VideoStreamItem? { if (SabrFormatSelector.video(info, itag) == null) return null @@ -88,6 +102,35 @@ private fun YoutubeSabrFormat.toVideoStreamItem(videoId: String, info: YoutubeSa ) } +private fun YoutubeSabrFormat.toAudioStreamItem(videoId: String): AudioStreamItem? { + val mime = mimeType?.takeIf { it.isNotBlank() } ?: return null + val container = mime.substringBefore(';').trim() + val track = audioTrackId?.takeIf { it.isNotBlank() } + ?.let { "&audioTrackId=${URLEncoder.encode(it, StandardCharsets.UTF_8)}" } + .orEmpty() + return AudioStreamItem( + url = "", + mimeType = container, + format = container.substringAfter('/').uppercase(), + bitrate = bitrate.takeIf { it > 0 }, + codec = mime.codec(), + quality = audioQuality, + itag = itag, + contentLength = contentLength.coerceAtLeast(0L), + initStart = 0L, + initEnd = 0L, + indexStart = 0L, + indexEnd = 0L, + audioTrackId = audioTrackId, + audioTrackName = audioTrackDisplayName, + audioLocale = audioTrackId?.substringBefore('.'), + isOriginal = isOriginalAudio, + deliveryMethod = SABR_DELIVERY_METHOD, + manifestUrl = "/sabr/manifest/$videoId", + sabrSessionUrl = "/sabr/session/$videoId?audioItag=$itag$track", + ) +} + private fun YoutubeSabrFormat.heightLabel(): String = height.takeIf { it > 0 }?.let { "${it}p" } ?: "" private fun String.formatName(): String = when (lowercase()) { diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt index 5672234b..40a67311 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrStreamContractFilterTest.kt @@ -87,6 +87,46 @@ class SabrStreamContractFilterTest { assertEquals(setOf("avc1.4d4028", "vp9", "av01.0.08M.08"), sabr.videoOnlyStreams.map { it.codec }.toSet()) } + @Test + fun `sabr contract exposes Opus audio for browsers without AAC MSE`() = runTest { + val aac = audioFormat(140, "audio/mp4; codecs=\"mp4a.40.2\"") + val opus = audioFormat(249, "audio/webm; codecs=\"opus\"") + val video = videoFormat(137, "video/mp4; codecs=\"avc1.4d4028\"") + val info = mockk { + every { formats } returns listOf(aac, opus, video) + every { findFormatByItag(any()) } answers { + formats.firstOrNull { it.itag == firstArg() } + } + } + val store = mockk() + coEvery { store.fetchInfo(VIDEO_ID, cachedFirst = true) } returns SabrPreparedInfo(info, null) + val aacStream = testAudioStream( + itag = 140, + deliveryMethod = "sabr", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?audioItag=140", + ) + val opusStream = aacStream.copy( + itag = 249, + codec = "opus", + mimeType = "audio/webm", + format = "WEBM", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?audioItag=249", + ) + val filtered = testStreamResponse( + videoOnlyStreams = listOf( + testVideoStream(itag = 137).copy( + deliveryMethod = "sabr", + sabrSessionUrl = "/sabr/session/$VIDEO_ID?videoItag=137", + ), + ), + audioStreams = listOf(aacStream), + ).withPlayableSabrStreams(YOUTUBE_URL, store).onlySabrStreams() + + assertEquals(setOf(140, 249), filtered.audioStreams.map { it.itag }.toSet()) + assertEquals("audio/webm", filtered.audioStreams.first { it.itag == 249 }.mimeType) + assertEquals("opus", filtered.audioStreams.first { it.itag == 249 }.codec) + } + private fun videoFormat(itag: Int, mime: String): YoutubeSabrFormat = mockk(relaxed = true) { every { isAudio } returns false @@ -98,6 +138,23 @@ class SabrStreamContractFilterTest { every { qualityLabel } returns "1080p" } + private fun audioFormat(itag: Int, mime: String): YoutubeSabrFormat = + mockk(relaxed = true) { + every { isAudio } returns true + every { isVideo } returns false + every { this@mockk.itag } returns itag + every { mimeType } returns mime + every { audioTrackId } returns null + every { isOriginalAudio } returns true + every { isDrc } returns false + every { xtags } returns null + every { bitrate } returns 128000 + every { width } returns 0 + every { height } returns 0 + every { qualityLabel } returns null + every { contentLength } returns 1_000_000L + } + private companion object { const val VIDEO_ID = "X4VbdwhkE10" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" From 8822eabdea2aeffa3ebd5e19ea3179532fb95ffc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 17:41:19 +0000 Subject: [PATCH 063/137] chore: update PipePipeExtractor dependency --- server-admin/build.gradle.kts | 2 +- server-domain/build.gradle.kts | 2 +- server-downloader/build.gradle.kts | 2 +- server-http/build.gradle.kts | 2 +- server-playback/build.gradle.kts | 2 +- server-sabr/build.gradle.kts | 2 +- server-services/build.gradle.kts | 2 +- server-token-gateway/build.gradle.kts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts index f0fe8083..a51af2ed 100644 --- a/server-admin/build.gradle.kts +++ b/server-admin/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("org.slf4j:slf4j-api:2.0.16") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts index b9b571aa..47f68667 100644 --- a/server-domain/build.gradle.kts +++ b/server-domain/build.gradle.kts @@ -7,7 +7,7 @@ dependencies { api(project(":server-core")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation(project(":server-cache")) implementation(project(":server-db")) implementation("org.jetbrains.exposed:exposed-core:1.5.0") diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts index 91d348be..521c9a9c 100644 --- a/server-downloader/build.gradle.kts +++ b/server-downloader/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts index 2f5a0a7b..023e827f 100644 --- a/server-http/build.gradle.kts +++ b/server-http/build.gradle.kts @@ -26,7 +26,7 @@ dependencies { implementation("io.ktor:ktor-utils-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts index acd7588b..4d05bd93 100644 --- a/server-playback/build.gradle.kts +++ b/server-playback/build.gradle.kts @@ -13,7 +13,7 @@ dependencies { implementation(project(":server-token-gateway")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts index bbcd7419..099629f1 100644 --- a/server-sabr/build.gradle.kts +++ b/server-sabr/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - api("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + api("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts index befc6f34..ec0d6606 100644 --- a/server-services/build.gradle.kts +++ b/server-services/build.gradle.kts @@ -24,7 +24,7 @@ dependencies { implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") implementation("org.slf4j:slf4j-api:2.0.16") diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts index 9bb1e6b1..0dad5922 100644 --- a/server-token-gateway/build.gradle.kts +++ b/server-token-gateway/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("org.slf4j:slf4j-api:2.0.16") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:a395a9ba16ae75987969ed9e7d330c928ad3bc20") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("io.ktor:ktor-client-core-jvm:3.5.2") From 91d01973e941b0d97a9b4f18a9ea4f94ed09ab94 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 17:41:19 +0000 Subject: [PATCH 064/137] ci: compile only affected server modules --- .github/scripts/ci-affected-tasks.sh | 15 ++++++++++----- .github/workflows/coverage.yml | 27 +++++++++++++++++++++++---- .github/workflows/docker.yml | 4 ++++ Dockerfile | 5 +++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/scripts/ci-affected-tasks.sh b/.github/scripts/ci-affected-tasks.sh index ab913acb..dc42fe49 100755 --- a/.github/scripts/ci-affected-tasks.sh +++ b/.github/scripts/ci-affected-tasks.sh @@ -10,6 +10,11 @@ before="${GITHUB_EVENT_BEFORE:-}" zero_sha="0000000000000000000000000000000000000000" sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" output_file="${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +task_name="${AFFECTED_GRADLE_TASK:-build}" +case "$task_name" in + build|test) ;; + *) echo "Unsupported affected Gradle task: $task_name" >&2; exit 2 ;; +esac emit_tasks() { if [[ -z "$1" ]]; then @@ -24,7 +29,7 @@ emit_tasks() { } if [[ "$event_name" != "push" && "$event_name" != "pull_request" ]]; then - emit_tasks "build" + emit_tasks "$task_name" exit 0 fi @@ -49,7 +54,7 @@ elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then git diff --name-only -z "HEAD^...$sha" ) else - emit_tasks "build" + emit_tasks "$task_name" exit 0 fi @@ -110,7 +115,7 @@ for file in "${changed_files[@]}"; do done if [[ "$full_build" == true ]]; then - emit_tasks "build" + emit_tasks "$task_name" exit 0 fi @@ -149,7 +154,7 @@ mark_affected() { for module in "${!changed_modules[@]}"; do if [[ ! -f "$module/build.gradle.kts" ]]; then echo "Changed module no longer exists: $module" >&2 - emit_tasks "build" + emit_tasks "$task_name" exit 0 fi @@ -158,7 +163,7 @@ done module_tasks="$( for module in "${!affected_modules[@]}"; do - printf ':%s:build\n' "$module" + printf ':%s:%s\n' "$module" "$task_name" done | sort )" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 79fbc5bf..c7f11e77 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,6 +16,8 @@ jobs: runs-on: [self-hosted, Linux, X64, ci-stack] steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - name: Isolate Gradle user home run: echo "GRADLE_USER_HOME=${RUNNER_TEMP}/gradle-user-home" >> "$GITHUB_ENV" @@ -28,18 +30,35 @@ jobs: - uses: gradle/actions/setup-gradle@v6 - - name: Run tests and coverage report - run: ./gradlew test jacocoTestReport jacocoTestCoverageVerification + - name: Detect affected test modules + id: affected + env: + AFFECTED_GRADLE_TASK: test + run: bash .github/scripts/ci-affected-tasks.sh + + - name: Run affected tests and coverage report + if: steps.affected.outputs.tasks != '' + env: + TASKS: ${{ steps.affected.outputs.tasks }} + run: | + readarray -t task_args <<<"$TASKS" + printf 'Running affected tests:\n%s\n' "$TASKS" + ./gradlew "${task_args[@]}" --parallel --max-workers=32 + ./gradlew jacocoTestReport jacocoTestCoverageVerification --parallel --max-workers=32 + + - name: No coverage required + if: steps.affected.outputs.tasks == '' + run: echo "No source or build changes require coverage." - name: Upload coverage HTML report - if: always() + if: always() && steps.affected.outputs.tasks != '' uses: actions/upload-artifact@v7 with: name: jacoco-html-report path: build/reports/jacoco/test/html - name: Upload coverage XML report - if: always() + if: always() && steps.affected.outputs.tasks != '' uses: actions/upload-artifact@v7 with: name: jacoco-xml-report diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a2f9b90c..d1c3232c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,10 @@ on: branches: - main - dev + paths-ignore: + - "**.md" + - "docs/**" + - ".github/ISSUE_TEMPLATE/**" tags: - "v*" workflow_dispatch: diff --git a/Dockerfile b/Dockerfile index d4e986b8..ece4c2c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM --platform=$BUILDPLATFORM eclipse-temurin:25-jdk-alpine AS builder ARG APP_VERSION=0.1.0 ARG GITHUB_SHA=unknown ARG BUILD_TIME=unknown +ARG GRADLE_MAX_WORKERS=32 ENV GITHUB_SHA=$GITHUB_SHA ENV BUILD_TIME=$BUILD_TIME WORKDIR /app @@ -24,8 +25,8 @@ COPY server-sabr/ ./server-sabr/ COPY server-test-support/ ./server-test-support/ COPY server-services/ ./server-services/ COPY server-token-gateway/ ./server-token-gateway/ -RUN ./gradlew dependencies --no-daemon -q || true -RUN ./gradlew shadowJar --no-daemon -q -PappVersion="$APP_VERSION" +RUN ./gradlew dependencies --no-daemon --parallel --max-workers="$GRADLE_MAX_WORKERS" -q || true +RUN ./gradlew shadowJar --no-daemon --parallel --max-workers="$GRADLE_MAX_WORKERS" -q -PappVersion="$APP_VERSION" FROM eclipse-temurin:25-jre-alpine AS runner RUN apk upgrade --no-cache \ From 2f1dd253f4e0e64dbdc48c12ff91a35fa2399aae Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 21 Sep 2026 19:27:36 +0000 Subject: [PATCH 065/137] fix: refresh SABR demand idle timestamp --- .../server/services/SabrSegmentDemandTracker.kt | 1 + .../services/SabrSegmentDemandResolutionTest.kt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt index 2a8c9f76..e7187035 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt @@ -105,6 +105,7 @@ fun SabrSessionHolder.requestSegmentDemand( generation: Long = activeGeneration(), registeredAtMs: Long = System.currentTimeMillis(), ): Unit = synchronized(this) { + touch() val state = playbackState() if (generation == activeGeneration() && state != SabrPlaybackState.TERMINAL && state != SabrPlaybackState.NETWORK_FAILED) { SabrSegmentDemandTracker.request(this, request, registeredAtMs) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt index d718d8df..e990af50 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSegmentDemandResolutionTest.kt @@ -6,6 +6,7 @@ import io.mockk.verify import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.SabrMediaHeader import dev.typetype.server.sabr.SabrMediaSegment @@ -17,6 +18,20 @@ import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant class SabrSegmentCacheTest { + @Test + fun `segment demand refreshes session idle timestamp`() { + val audio = format(140, isAudio = true) + val video = format(137, isAudio = false) + val holder = holder(audio, video) + val stale = holder.lastRequestAt + val request = SabrSegmentRequest.media(audio, 1) + + holder.requestSegmentDemand(request) + + assertTrue(holder.lastRequestAt.isAfter(stale)) + SabrSegmentDemandTracker.clearAll() + } + @Test fun `vod cache observes media without copying segment bytes`() { val segmentCache = SabrSegmentCache() @@ -57,6 +72,8 @@ class SabrSegmentCacheTest { val session = mockk() val state = mockk() every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.isBeyondEnd(any()) } returns false every { state.setActiveTrackTypes(any(), any()) } returns Unit return SabrSessionHolder( session = session, From 36c2dcd249164342ed8bce13a26c922786cad391 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 09:52:43 +0000 Subject: [PATCH 066/137] fix: remove global BiliBili cookie configuration --- .env.example | 4 ---- .../server/services/BilibiliCookieConfig.kt | 10 +++------- .../server/services/NewPipeInitializer.kt | 20 ------------------- .../services/BilibiliCookieConfigTest.kt | 16 --------------- 4 files changed, 3 insertions(+), 47 deletions(-) diff --git a/.env.example b/.env.example index 7fadb293..0c395a83 100644 --- a/.env.example +++ b/.env.example @@ -6,10 +6,6 @@ DRAGONFLY_URL=redis://dragonfly:6379 DOWNLOADER_SERVICE_URL=http://typetype-downloader:18093 YOUTUBE_SESSION_ENCRYPTION_KEY=replace-with-at-least-32-random-characters YOUTUBE_OUTBOUND_PROXY_URL= -# Optional BiliBili account cookie header. Keep the value out of git and logs. -# BILIBILI_COOKIE=SESSDATA=...; bili_jct=...; buvid3=... -# Alternatively provide the cookie through a mounted secret file: -# BILIBILI_COOKIE_FILE=/run/secrets/bilibili_cookie YOUTUBE_REMOTE_LOGIN_SERVICE_URL=http://typetype-token:8081 YOUTUBE_REMOTE_LOGIN_CALLBACK_BASE_URL=http://typetype-server:8080 YOUTUBE_REMOTE_LOGIN_INTERNAL_TOKEN=replace-with-shared-internal-token diff --git a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt index 5719a28e..454496ab 100644 --- a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/BilibiliCookieConfig.kt @@ -1,9 +1,8 @@ package dev.typetype.server.services -/** Validates the optional instance-wide BiliBili cookie used by extraction. */ +/** Validates cookies captured for an authenticated BiliBili user session. */ data class BilibiliCookieConfig internal constructor( val cookieHeader: String?, - val wasProvided: Boolean, ) { val isConfigured: Boolean get() = cookieHeader != null @@ -13,13 +12,10 @@ data class BilibiliCookieConfig internal constructor( private val COOKIE_NAME = Regex("[!#\\$%&'*+.^_`|~0-9A-Za-z-]+") private val ACCOUNT_COOKIE_NAMES = setOf("SESSDATA", "bili_jct", "buvid3", "DedeUserID") - fun fromEnvironment(read: (String) -> String? = System::getenv): BilibiliCookieConfig = - fromRaw(SecretConfigReader.read("BILIBILI_COOKIE", read)) - fun fromRaw(raw: String?): BilibiliCookieConfig { val supplied = raw?.trim()?.takeIf { it.isNotEmpty() } - ?: return BilibiliCookieConfig(cookieHeader = null, wasProvided = false) - return BilibiliCookieConfig(cookieHeader = normalize(supplied), wasProvided = true) + ?: return BilibiliCookieConfig(cookieHeader = null) + return BilibiliCookieConfig(cookieHeader = normalize(supplied)) } private fun normalize(raw: String): String? { diff --git a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt index 2d9493fb..c1770226 100644 --- a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/NewPipeInitializer.kt @@ -1,25 +1,19 @@ package dev.typetype.server.services -import dev.typetype.server.downloader.BilibiliCookieContext import dev.typetype.server.downloader.OkHttpDownloader import org.schabi.newpipe.extractor.NewPipe -import org.schabi.newpipe.extractor.ServiceList import org.schabi.newpipe.extractor.services.youtube.YoutubeApiDecoder -import org.slf4j.LoggerFactory import java.net.ProxySelector object NewPipeInitializer { @Volatile private var initialized = false @Volatile private var decoderServiceUrl: String? = null - private val log = LoggerFactory.getLogger(NewPipeInitializer::class.java) @Synchronized fun init( tokenServiceUrl: String? = null, youtubeProxySelector: ProxySelector? = null, - bilibiliCookie: BilibiliCookieConfig = BilibiliCookieConfig.fromEnvironment(), ): Unit { - configureBilibili(bilibiliCookie) NewPipe.setYoutubePlayerClient(YOUTUBE_PLAYER_CLIENT) val normalizedUrl = tokenServiceUrl?.trim()?.takeIf { it.isNotBlank() } if (normalizedUrl != null && normalizedUrl != decoderServiceUrl) { @@ -38,19 +32,5 @@ object NewPipeInitializer { NewPipe.setYoutubeSessionPoTokenProvider(TypetypeYoutubeSessionPoTokenProvider) } - private fun configureBilibili(config: BilibiliCookieConfig) { - ServiceList.BiliBili.setTokens(config.cookieHeader.orEmpty()) - ServiceList.BiliBili.setCookieFunctions( - if (config.isConfigured) BILIBILI_COOKIE_FUNCTIONS else emptySet(), - ) - BilibiliCookieContext.set(config.cookieHeader) - if (config.wasProvided && !config.isConfigured) { - log.warn("Ignoring invalid BILIBILI_COOKIE; expected a cookie header containing an account cookie") - } else if (config.isConfigured) { - log.info("BiliBili account cookie configured for instance extraction") - } - } - private const val YOUTUBE_PLAYER_CLIENT = "mweb" - private val BILIBILI_COOKIE_FUNCTIONS = setOf("high_res", "ai_subtitle") } diff --git a/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt index 11a13336..5ebba5ea 100644 --- a/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt +++ b/server-token-gateway/src/test/kotlin/dev/typetype/server/services/BilibiliCookieConfigTest.kt @@ -5,7 +5,6 @@ import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import java.nio.file.Files class BilibiliCookieConfigTest { @Test @@ -14,7 +13,6 @@ class BilibiliCookieConfigTest { "Cookie: SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", ) - assertTrue(config.wasProvided) assertTrue(config.isConfigured) assertEquals( "SESSDATA=session; bili_jct=csrf; buvid3=device; theme=dark", @@ -22,20 +20,6 @@ class BilibiliCookieConfigTest { ) } - @Test - fun `reads a mounted cookie file through the secret convention`() { - val file = Files.createTempFile("typetype-bilibili", ".cookie") - try { - Files.writeString(file, "SESSDATA=from-file; bili_jct=csrf") - val config = BilibiliCookieConfig.fromEnvironment { name -> - if (name == "BILIBILI_COOKIE_FILE") file.toString() else null - } - assertEquals("SESSDATA=from-file; bili_jct=csrf", config.cookieHeader) - } finally { - Files.deleteIfExists(file) - } - } - @Test fun `rejects malformed and unrelated cookies`() { assertFalse(BilibiliCookieConfig.fromRaw("theme=dark").isConfigured) From 13ff6adbf19b8fcf2a1a86466e785a293863d714 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 13:34:26 +0000 Subject: [PATCH 067/137] fix: protect active live SABR sessions --- .../server/services/SabrSessionRegistry.kt | 70 ++++++++++++++++--- .../server/services/SabrSessionStore.kt | 11 +-- .../server/services/SabrSessionStoreTest.kt | 14 ++++ 3 files changed, 82 insertions(+), 13 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt index 263bb960..beae5244 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionRegistry.kt @@ -2,11 +2,13 @@ package dev.typetype.server.services import java.time.Instant import java.util.concurrent.ConcurrentHashMap +import org.slf4j.LoggerFactory class SabrSessionRegistry { private val sessions = ConcurrentHashMap() private val sessionsByToken = ConcurrentHashMap() private val mutationLock = Any() + private val logger = LoggerFactory.getLogger(SabrSessionRegistry::class.java) fun get(key: SabrSessionKey): SabrSessionHolder? { val holder = sessions[key] @@ -76,31 +78,47 @@ class SabrSessionRegistry { return holder } - fun ensureCapacity(maxSessions: Int) { + fun ensureCapacity( + maxSessions: Int, + activeCutoff: Instant = Instant.now().minus(SabrSessionStoreDefaults.idleEviction()), + ) { while (sessions.size >= maxSessions) { val oldest = sessions.entries + .asSequence() + .filterNot { isActiveLive(it.value, activeCutoff) } .minByOrNull { it.value.lastRequestAt } - ?: return - remove(oldest.key) + if (oldest == null) { + logCapacitySaturated(maxSessions) + return + } + evict(oldest.key, "capacity") } } - fun trimToCapacity(maxSessions: Int, protected: SabrSessionHolder) { + fun trimToCapacity( + maxSessions: Int, + protected: SabrSessionHolder, + activeCutoff: Instant = Instant.now().minus(SabrSessionStoreDefaults.idleEviction()), + ) { while (sessions.size > maxSessions) { val oldest = sessions.entries .asSequence() .filterNot { it.value === protected } + .filterNot { isActiveLive(it.value, activeCutoff) } .minByOrNull { it.value.lastRequestAt } - ?: return - remove(oldest.key) + if (oldest == null) { + logCapacitySaturated(maxSessions) + return + } + evict(oldest.key, "capacity") } } fun evictIdle(cutoff: Instant) { val stale = sessions.entries - .filter { it.value.lastRequestAt.isBefore(cutoff) } + .filter { it.value.lastRequestAt.isBefore(cutoff) && !isActiveLive(it.value, cutoff) } .map { it.key } - stale.forEach(::remove) + stale.forEach { evict(it, "idle") } } fun clear() { @@ -122,4 +140,40 @@ class SabrSessionRegistry { } holder?.releaseResources() } + + private fun isActiveLive(holder: SabrSessionHolder, cutoff: Instant): Boolean { + if (!holder.expectsLive() || holder.lastRequestAt.isBefore(cutoff)) return false + return holder.playbackState() !in setOf( + SabrPlaybackState.NETWORK_FAILED, + SabrPlaybackState.TERMINAL, + SabrPlaybackState.STOPPED, + ) + } + + private fun evict(key: SabrSessionKey, reason: String) { + val now = Instant.now() + val holder = synchronized(mutationLock) { + sessions.remove(key)?.also { sessionsByToken.remove(it.sessionToken, it) } + } ?: return + val ageMs = java.time.Duration.between(holder.lastRequestAt, now).toMillis().coerceAtLeast(0L) + holder.releaseResources() + logger.info( + "sabr_session event=evicted reason={} videoId={} ageMs={} registrySize={} liveSessions={} state={}", + reason, + holder.key.videoId, + ageMs, + sessions.size, + sessions.values.count { it.expectsLive() }, + holder.playbackState(), + ) + } + + private fun logCapacitySaturated(maxSessions: Int) { + logger.warn( + "sabr_session event=capacity_saturated maxSessions={} registrySize={} liveSessions={}", + maxSessions, + sessions.size, + sessions.values.count { it.expectsLive() }, + ) + } } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index e631d171..81b629bd 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -71,7 +71,8 @@ class SabrSessionStore( isolatedSourceId, ) registry.getReusable(key)?.let { return it } - registry.ensureCapacity(maxSessions) + val activeCutoff = Instant.now().minus(idleEviction) + registry.ensureCapacity(maxSessions, activeCutoff) val holder = sessionFactory.create( key, info, @@ -84,7 +85,7 @@ class SabrSessionStore( ) val active = registry.put(key, holder) if (active !== holder) return active - registry.trimToCapacity(maxSessions, holder) + registry.trimToCapacity(maxSessions, holder, activeCutoff) if (startPump) startPump(holder) return holder } @@ -125,6 +126,7 @@ class SabrSessionStore( .takeIf { cached -> cached.all { it != null } }?.filterNotNull() suspend fun cachedSegment(holder: SabrSessionHolder, request: SabrSegmentRequest): CachedSabrSegment? { + holder.touch() holder.session.getCachedSegment(request)?.let { segmentCache.put(holder, it) holder.clearSegmentDemand(request) @@ -166,11 +168,10 @@ class SabrSessionStore( suspend fun fetchSegment( holder: SabrSessionHolder, request: SabrSegmentRequest, - ): SabrMediaSegment? = pump.fetchSegment(holder, request) + ): SabrMediaSegment? = holder.also { it.touch() }.let { pump.fetchSegment(it, request) } suspend fun fetchMediaAt(holder: SabrSessionHolder, playerTimeMs: Long): List? = - pump.fetchMediaAt(holder, playerTimeMs) - + holder.also { it.touch() }.let { pump.fetchMediaAt(it, playerTimeMs) } suspend fun fetchInitializationData( holder: SabrSessionHolder, format: YoutubeSabrFormat, diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt index f1569bc7..aae658f8 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrSessionStoreTest.kt @@ -31,6 +31,20 @@ class SabrSessionRegistryTest { assertSame(idle, registry.get(idleKey)) } + @Test + fun `capacity eviction keeps a recently active live session`() { + val registry = SabrSessionRegistry() + val live = holder("live", Instant.now()).also { it.markExpectedLive() } + val idle = holder("idle", Instant.EPOCH) + registry.put(live.key, live) + registry.put(idle.key, idle) + + registry.ensureCapacity(2, Instant.now().minusSeconds(60)) + + assertSame(live, registry.get(live.key)) + assertNull(registry.get(idle.key)) + } + @Test fun `idle eviction removes stale sessions`() { val registry = SabrSessionRegistry() From 7aa81f4c7698fd4a548b686af188838fb73ed002 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 14:03:47 +0000 Subject: [PATCH 068/137] style: keep SABR session store within line limit --- .../main/kotlin/dev/typetype/server/services/SabrSessionStore.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 81b629bd..40976939 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -92,7 +92,6 @@ class SabrSessionStore( fun startPump(holder: SabrSessionHolder) { scope.launchSabrPump(pump, registry, holder, pumpLoopIntervalMs) } - fun warmPlaybackAsync(holder: SabrSessionHolder) { if (holder.playerTimeMs() > 0L) return if (holder.playbackState() == SabrPlaybackState.REQUESTING || holder.playbackState() == SabrPlaybackState.REPOSITIONING) return From 6c6fc0f19ec8335f8e4b90df7d361f58e03fbf44 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 15:08:49 +0000 Subject: [PATCH 069/137] fix: widen live SABR read-ahead cushion --- .../server/services/SabrPumpPolicy.kt | 1 + .../server/services/SabrPumpRuntime.kt | 23 +++++++++++++-- .../server/services/SabrPumpPolicyTest.kt | 29 +++++++++++++++++++ .../services/SabrRandomAccessProbeTest.kt | 1 + 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt index 3b4f090d..83264102 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt @@ -12,6 +12,7 @@ object SabrPumpPolicy { const val SEEK_MODE_MS = 8_000L const val MIN_SERVER_READAHEAD_CUSHION_MS = 3_000L const val SERVER_AHEAD_MARGIN_MS = 16_000L + const val LIVE_READAHEAD_CUSHION_MS = 90_000L const val DEMAND_TARGET_DEADLINE_MS = 15_000L const val MAX_DEMAND_BACKOFF_EXTENSION_MS = 30_000L const val MAX_PROTECTED_NO_MEDIA_RESPONSES = 5 diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt index fae2144b..e1b22ca2 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt @@ -62,14 +62,31 @@ class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) } fun targetReadaheadCushionMs(holder: SabrSessionHolder): Long { + var usesLiveReadAhead = false val baseCushionMs = when { isSeekMode() -> SabrPumpPolicy.SEEK_READAHEAD_CUSHION_MS isStartupBurst() -> SabrPumpPolicy.STARTUP_BURST_READAHEAD_CUSHION_MS holder.playerTimeMs() == 0L && holder.readerTailMs() == 0L -> SabrPumpPolicy.STARTUP_READAHEAD_CUSHION_MS + holder.livePlaybackSnapshot()?.active == true -> { + usesLiveReadAhead = true + liveReadaheadCushionMs(holder) + } else -> serverReadaheadCushionMs(holder) } - return rateAwareCushionMs(baseCushionMs, holder.playbackRate()) + val maximumCushionMs = if (usesLiveReadAhead) baseCushionMs else MAX_RATE_AWARE_CUSHION_MS + return rateAwareCushionMs(baseCushionMs, holder.playbackRate(), maximumCushionMs) + } + + private fun liveReadaheadCushionMs(holder: SabrSessionHolder): Long { + val bitsPerSecond = holder.videoFormat.bitrate.toLong().coerceAtLeast(0L) + + holder.audioFormat.bitrate.toLong().coerceAtLeast(0L) + if (bitsPerSecond <= 0L) return SabrPumpPolicy.LIVE_READAHEAD_CUSHION_MS + val bytesPerMs = (bitsPerSecond / 8_000L).coerceAtLeast(1L) + val memoryBoundMs = SabrPumpPolicy.MAX_AHEAD_BYTES / bytesPerMs + return SabrPumpPolicy.LIVE_READAHEAD_CUSHION_MS.coerceAtMost( + memoryBoundMs.coerceAtLeast(SabrPumpPolicy.MIN_SERVER_READAHEAD_CUSHION_MS), + ) } private fun serverReadaheadCushionMs(holder: SabrSessionHolder): Long { @@ -82,8 +99,8 @@ class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) ) } - private fun rateAwareCushionMs(baseMs: Long, playbackRate: Float): Long = - (baseMs * playbackRate.coerceIn(0.25f, 4.0f)).roundToLong().coerceAtMost(MAX_RATE_AWARE_CUSHION_MS) + private fun rateAwareCushionMs(baseMs: Long, playbackRate: Float, maximumMs: Long): Long = + (baseMs * playbackRate.coerceIn(0.25f, 4.0f)).roundToLong().coerceAtMost(maximumMs) private fun cappedServerAheadPlayerTimeMs(holder: SabrSessionHolder, edgeMs: Long): Long = maxOf(holder.playerTimeMs(), edgeMs - SabrPumpPolicy.SERVER_AHEAD_MARGIN_MS) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt index 9f021795..d2d08b4b 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt @@ -19,10 +19,38 @@ import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) class SabrPumpLauncherTest { + @Test + fun `active live playback keeps a bounded larger read ahead cushion`() { + val audio = format(140, isAudio = true) + val video = format(136, isAudio = false) + val state = mockk(relaxed = true) + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.isLive } returns true + every { state.isLive } returns true + every { state.isPostLiveDvr } returns false + val holder = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session-token", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + holder.setPlayerTimeMs(100_000L) + var now = 0L + val runtime = SabrPumpRuntime { now } + now = 30_000L + + assertEquals(90_000L, runtime.targetReadaheadCushionMs(holder)) + } + @Test fun `watchdog ignores companion progress for missing demand`() = runTest { SabrSegmentDemandTracker.clearAll() @@ -226,6 +254,7 @@ class SabrPumpLauncherTest { val format = mockk() every { format.itag } returns itag every { format.isAudio } returns isAudio + every { format.bitrate } returns if (isAudio) 128_000 else 2_000_000 return format } } diff --git a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt index b0440c56..ed54c1d1 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrRandomAccessProbeTest.kt @@ -115,6 +115,7 @@ class SabrPumpRuntimeTest { val session = mockk() val state = mockk() every { holder.session } returns session + every { holder.expectsLive() } returns false every { holder.playerTimeMs() } returns playerTimeMs every { holder.playbackRate() } returns playbackRate every { holder.readerTailMs() } returns 1L From 87ff110de212b72cc023668d5aaaccaf7a1ab998 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 18:56:17 +0000 Subject: [PATCH 070/137] fix: track live SABR demand interruptions --- .../server/services/SabrDemandDeadline.kt | 11 ++- .../services/SabrInFlightDemandTracker.kt | 99 +++++++++++++++++++ .../server/services/SabrPumpLogger.kt | 33 ++++++- .../services/SabrSegmentDemandTracker.kt | 19 ++++ 4 files changed, 157 insertions(+), 5 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt index de717f30..beb01df8 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandDeadline.kt @@ -15,12 +15,19 @@ class SabrDemandDeadline( registeredAtMs: Long, nowMs: Long, backoffRemainingMs: Long, - ): Boolean { + ): Boolean = nowMs >= deadlineAtMs(identity, registeredAtMs, nowMs, backoffRemainingMs) + + fun deadlineAtMs( + identity: String, + registeredAtMs: Long, + nowMs: Long, + backoffRemainingMs: Long, + ): Long { if (this.identity != identity || this.registeredAtMs != registeredAtMs) { reset(identity, registeredAtMs) } extendForBackoff(nowMs, backoffRemainingMs) - return nowMs >= expiresAtMs + return expiresAtMs } private fun reset(identity: String, registeredAtMs: Long) { diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt index 0b73deed..d12ea1f0 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInFlightDemandTracker.kt @@ -2,6 +2,18 @@ package dev.typetype.server.services import dev.typetype.server.sabr.SabrSegmentRequest import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +internal enum class SabrDemandInterruptionAction { + WAIT_FOR_LIVE, + RESTART, +} + +internal data class SabrDemandInterruption( + val action: SabrDemandInterruptionAction, + val atMs: Long, + val reason: String, +) class SabrInFlightDemand( val request: SabrSegmentRequest, @@ -11,6 +23,11 @@ class SabrInFlightDemand( ) { private var lastProgressVersion = Long.MIN_VALUE private var lastProgressAtMs = registeredAtMs + private var attemptCancellation: (() -> Unit)? = null + private var interruption: SabrDemandInterruption? = null + private val attemptCount = AtomicInteger(0) + @Volatile private var attemptStartedAtNanos = 0L + @Volatile private var lastAttemptDurationMs = -1L fun observeProgress(version: Long, observedAtMs: Long): Long { if (version != lastProgressVersion) { @@ -19,6 +36,53 @@ class SabrInFlightDemand( } return lastProgressAtMs } + + fun registerAttemptCancellation(cancel: () -> Unit): Unit { + val cancelNow = synchronized(this) { + if (interruption != null) true else { + attemptCancellation = cancel + false + } + } + if (cancelNow) cancel() + } + + internal fun requestInterruption(next: SabrDemandInterruption): Boolean { + val callback = synchronized(this) { + if (interruption != null) return@synchronized null + interruption = next + val pending = attemptCancellation + attemptCancellation = null + pending + } + callback?.invoke() + return true + } + + internal fun consumeInterruption(): SabrDemandInterruption? = synchronized(this) { + interruption.also { interruption = null } + } + + fun clearAttemptCancellation(): Unit = synchronized(this) { + attemptCancellation = null + } + + fun recordAttemptStarted(): Unit { + attemptCount.incrementAndGet() + attemptStartedAtNanos = System.nanoTime() + } + + fun recordAttemptFinished(): Unit { + val startedAt = attemptStartedAtNanos + if (startedAt > 0L) { + lastAttemptDurationMs = ((System.nanoTime() - startedAt) / 1_000_000L).coerceAtLeast(0L) + } + attemptStartedAtNanos = 0L + } + + fun attempts(): Int = attemptCount.get() + + fun lastAttemptDurationMs(): Long = lastAttemptDurationMs } object SabrInFlightDemandTracker { @@ -69,3 +133,38 @@ fun SabrSessionHolder.finishInFlightSegmentDemand(identity: String): Boolean = fun SabrSessionHolder.clearInFlightSegmentDemand(): Unit = SabrInFlightDemandTracker.clear(this) + +internal fun SabrSessionHolder.registerInFlightDemandCancellation( + identity: String, + cancel: () -> Unit, +): Boolean = SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.let { it.registerAttemptCancellation(cancel); true } + ?: false + +internal fun SabrSessionHolder.requestInFlightDemandInterruption( + identity: String, + interruption: SabrDemandInterruption, +): Boolean = SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.requestInterruption(interruption) + ?: false + +internal fun SabrSessionHolder.consumeInFlightDemandInterruption(identity: String): SabrDemandInterruption? = + SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.consumeInterruption() + +internal fun SabrSessionHolder.clearInFlightDemandCancellation(identity: String): Unit { + SabrInFlightDemandTracker.current(this) + ?.takeIf { it.identity == identity } + ?.clearAttemptCancellation() +} + +internal fun SabrSessionHolder.recordInFlightAttemptStarted(): Unit { + SabrInFlightDemandTracker.current(this)?.recordAttemptStarted() +} + +internal fun SabrSessionHolder.recordInFlightAttemptFinished(): Unit { + SabrInFlightDemandTracker.current(this)?.recordAttemptFinished() +} diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt index 16531eba..532b1876 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpLogger.kt @@ -52,15 +52,42 @@ object SabrPumpLogger { ) } - fun expired(holder: SabrSessionHolder, request: SabrSegmentRequest, recoverable: Boolean): Unit { + fun expired( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + recoverable: Boolean, + registeredAtMs: Long? = null, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long? = null, + reason: String = "deadline", + attempts: Int = 0, + lastAttemptDurationMs: Long = -1L, + event: String = "demand_expired", + ): Unit { + val liveHeadTimeMs = holder.livePlaybackSnapshot()?.headTimeMs ?: -1L + val segmentEndMs = runCatching { holder.playbackSegmentEndMs(request.format, request.sequenceNumber) } + .getOrDefault(-1L) + val bufferedEdgeMs = holder.session.streamState.getMinBufferedEndMs() + val ageMs = registeredAtMs?.let { (nowMs - it).coerceAtLeast(0L) } ?: -1L logger.warn( - "sabr_pump event=demand_expired videoId={} request={} recoverable={} state={} requestNumber={} edgeMs={} readerHeadMs={} readerTailMs={} cachedBytes={}", + "sabr_pump event={} videoId={} request={} track={} sequence={} recoverable={} ageMs={} registeredAtMs={} expectedDelayMs={} liveHeadTimeMs={} segmentEndMs={} bufferedEdgeMs={} lastAttemptDurationMs={} attempts={} reason={} state={} requestNumber={} readerHeadMs={} readerTailMs={} cachedBytes={}", + event, holder.key.videoId, request.summary(), + if (request.format.isAudio) "audio" else "video", + request.sequenceNumber, recoverable, + ageMs, + registeredAtMs ?: -1L, + expectedDelayMs ?: -1L, + liveHeadTimeMs, + segmentEndMs, + bufferedEdgeMs, + lastAttemptDurationMs, + attempts, + reason, holder.playbackState(), holder.session.requestNumber, - holder.session.streamState.getMinBufferedEndMs(), holder.readerHeadMs(), holder.readerTailMs(), holder.session.cachedBytes, diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt index e7187035..864bc979 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSegmentDemandTracker.kt @@ -72,6 +72,19 @@ object SabrSegmentDemandTracker { return demand.registeredAtMs.takeIf { identity(requestKey, demand) == identity } } + fun requeue( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + registeredAtMs: Long, + ): Boolean { + val requestKey = key(holder, request) + val demand = demands[requestKey] ?: return false + if (identity(requestKey, demand) != identity) return false + val replacement = SegmentDemand(request, order.incrementAndGet(), registeredAtMs) + return demands.replace(requestKey, demand, replacement) + } + fun clear(holder: SabrSessionHolder, request: SabrSegmentRequest, identity: String): Boolean { val requestKey = key(holder, request) val demand = demands[requestKey] ?: return false @@ -128,6 +141,12 @@ fun SabrSessionHolder.isSegmentDemandActive(request: SabrSegmentRequest, identit fun SabrSessionHolder.segmentDemandRegisteredAtMs(request: SabrSegmentRequest, identity: String): Long? = SabrSegmentDemandTracker.registeredAtMs(this, request, identity) +internal fun SabrSessionHolder.requeueSegmentDemand( + request: SabrSegmentRequest, + identity: String, + registeredAtMs: Long, +): Boolean = SabrSegmentDemandTracker.requeue(this, request, identity, registeredAtMs) + fun SabrSessionHolder.clearSegmentDemand(request: SabrSegmentRequest, identity: String): Boolean = SabrSegmentDemandTracker.clear(this, request, identity) From 6f420acf08264f79dd268d5bb2502d6f6907c9ca Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 18:56:24 +0000 Subject: [PATCH 071/137] fix: requeue future live SABR demands --- .../server/services/SabrDemandAttempt.kt | 90 +++++++++++++++++++ .../services/SabrDemandAttemptFinisher.kt | 77 +++++++++++++++- .../server/services/SabrDemandWatchdog.kt | 63 +++++++++---- .../server/services/SabrLivePumpStep.kt | 3 +- .../server/services/SabrSessionPumpLoop.kt | 26 ++---- 5 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt new file mode 100644 index 00000000..1e7e109f --- /dev/null +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttempt.kt @@ -0,0 +1,90 @@ +package dev.typetype.server.services + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrSession + +internal suspend fun runSabrDemandAttempt( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + wasFutureLiveRequest: Boolean, + runtime: SabrPumpRuntime, + pump: suspend () -> YoutubeSabrSession.DemandResponseResult, + onResolved: (SabrMediaSegment) -> Unit, +): Boolean { + if (!holder.beginInFlightSegmentDemand(request, identity, wasFutureLiveRequest)) return true + try { + SabrPumpLogger.start(holder, "demand", request) + runtime.beginDemand(identity) + return coroutineScope { + val attempt = async(start = CoroutineStart.UNDISPATCHED) { + holder.recordInFlightAttemptStarted() + try { + pump() + } finally { + holder.recordInFlightAttemptFinished() + } + } + holder.registerInFlightDemandCancellation(identity) { + attempt.cancel(CancellationException("SABR demand interrupted")) + } + try { + val result = attempt.await() + val interruption = holder.consumeInFlightDemandInterruption(identity) + if (interruption != null) { + return@coroutineScope finishInterruptedDemand(holder, request, identity, runtime, interruption) + } + SabrDemandAttemptFinisher.finish( + holder, + request, + identity, + result, + runtime, + wasFutureLiveRequest, + onResolved = { onResolved(it) }, + ) + } catch (error: CancellationException) { + val interruption = holder.consumeInFlightDemandInterruption(identity) + ?: throw error + finishInterruptedDemand(holder, request, identity, runtime, interruption) + } finally { + holder.clearInFlightDemandCancellation(identity) + } + } + } finally { + holder.finishInFlightSegmentDemand(identity) + } +} + +private fun finishInterruptedDemand( + holder: SabrSessionHolder, + request: SabrSegmentRequest, + identity: String, + runtime: SabrPumpRuntime, + interruption: SabrDemandInterruption, +): Boolean = synchronized(holder) { + runtime.finishDemand(identity) + when (interruption.action) { + SabrDemandInterruptionAction.WAIT_FOR_LIVE -> { + if (holder.isSegmentDemandActive(request, identity)) { + holder.requeueSegmentDemand(request, identity, interruption.atMs) + } + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + false + } + + SabrDemandInterruptionAction.RESTART -> { + if (holder.playbackState() != SabrPlaybackState.TERMINAL && + holder.playbackState() != SabrPlaybackState.NETWORK_FAILED + ) { + holder.setPlaybackState(SabrPlaybackState.IDLE) + } + true + } + } +} diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index 14693b63..4b3f1c8d 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -11,14 +11,59 @@ object SabrDemandAttemptFinisher { if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false if (holder.session.getCachedSegment(demand.request) == null) return@synchronized false + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.RESTART, + atMs = System.currentTimeMillis(), + reason = "completed_demand", + ), + ) holder.setPlaybackState(SabrPlaybackState.IDLE) true } + fun interruptStalledInFlightDemand( + holder: SabrSessionHolder, + demand: SabrInFlightDemand, + nowMs: Long, + expectedDelayMs: Long, + ): Boolean = synchronized(holder) { + val state = holder.playbackState() + if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false + if (!demand.futureLiveRequest || holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false + if (holder.session.getCachedSegment(demand.request) != null) return@synchronized false + if (!holder.requeueSegmentDemand(demand.request, demand.identity, nowMs)) return@synchronized false + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.WAIT_FOR_LIVE, + atMs = nowMs, + reason = "future_live_not_published", + ), + ) + SabrPumpLogger.expired( + holder, + demand.request, + recoverable = true, + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = "future_live_not_published", + attempts = demand.attempts(), + lastAttemptDurationMs = demand.lastAttemptDurationMs(), + event = "demand_requeued", + ) + true + } + fun expireStalledInFlightDemand( holder: SabrSessionHolder, demand: SabrInFlightDemand, recoverable: Boolean = demand.futureLiveRequest, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, ): Boolean = synchronized(holder) { val state = holder.playbackState() @@ -26,7 +71,25 @@ object SabrDemandAttemptFinisher { if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false holder.clearSegmentDemands() val message = "SABR demand stalled for ${demand.request.summary()}" - SabrPumpLogger.expired(holder, demand.request, recoverable) + holder.requestInFlightDemandInterruption( + demand.identity, + SabrDemandInterruption( + action = SabrDemandInterruptionAction.RESTART, + atMs = nowMs, + reason = "terminal_deadline", + ), + ) + SabrPumpLogger.expired( + holder, + demand.request, + recoverable, + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = "terminal_deadline", + attempts = demand.attempts(), + lastAttemptDurationMs = demand.lastAttemptDurationMs(), + ) holder.failTerminal(if (recoverable) sabrRecoverableFailureMessage(message) else message) true } @@ -36,12 +99,22 @@ object SabrDemandAttemptFinisher { request: SabrSegmentRequest, identity: String, recoverable: Boolean = false, + nowMs: Long = System.currentTimeMillis(), + expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, ): Boolean = synchronized(holder) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false val current = holder.nextSegmentDemand() ?: return@synchronized false if (!current.matches(request) || holder.segmentDemandIdentity(current) != identity) return@synchronized false - SabrPumpLogger.expired(holder, request, recoverable) + SabrPumpLogger.expired( + holder, + request, + recoverable, + registeredAtMs = holder.segmentDemandRegisteredAtMs(request, identity), + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = "terminal_deadline", + ) fail(holder, request, identity, recoverable) } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index faa6bdb2..cfaf27f5 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -13,29 +13,48 @@ class SabrDemandWatchdog( if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return false val inFlightDemand = holder.inFlightSegmentDemand() if (inFlightDemand != null) { + val nowMs = clock() + val backoffRemainingMs = holder.session.demandBackoffRemainingMs if (holder.isLiveDemandOutsideRecoverableWindow(inFlightDemand.request) && - SabrDemandAttemptFinisher.expireStalledInFlightDemand(holder, inFlightDemand, recoverable = true) + SabrDemandAttemptFinisher.expireStalledInFlightDemand( + holder, + inFlightDemand, + recoverable = true, + nowMs = nowMs, + ) ) { return true } if (inFlightDemand.futureLiveRequest) holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) - val nowMs = clock() - val backoffRemainingMs = holder.session.demandBackoffRemainingMs val lastProgressAtMs = inFlightDemand.observeProgress(holder.session.mediaProgressVersion, nowMs) val completedIdle = holder.session.getCachedSegment(inFlightDemand.request) != null && nowMs - lastProgressAtMs >= SabrPumpPolicy.COMPLETED_DEMAND_IDLE_MS if (completedIdle && SabrDemandAttemptFinisher.interruptCompletedInFlightDemand(holder, inFlightDemand)) { return true } - if (deadline.isExpired( - inFlightDemand.identity, - inFlightDemand.registeredAtMs, - nowMs, - backoffRemainingMs, - ) && - SabrDemandAttemptFinisher.expireStalledInFlightDemand(holder, inFlightDemand) - ) { - return true + val deadlineAtMs = deadline.deadlineAtMs( + inFlightDemand.identity, + inFlightDemand.registeredAtMs, + nowMs, + backoffRemainingMs, + ) + if (nowMs >= deadlineAtMs) { + if (inFlightDemand.futureLiveRequest) { + SabrDemandAttemptFinisher.interruptStalledInFlightDemand( + holder, + inFlightDemand, + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + ) + } else if (SabrDemandAttemptFinisher.expireStalledInFlightDemand( + holder, + inFlightDemand, + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + ) + ) { + return true + } } delay(nextCheckDelayMs(backoffRemainingMs, inFlightDemand.futureLiveRequest)) continue @@ -57,18 +76,26 @@ class SabrDemandWatchdog( } if (futureLiveRequest) { holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + delay(nextCheckDelayMs(holder.session.demandBackoffRemainingMs, futureLiveRequest)) + continue } val identity = holder.segmentDemandIdentity(request) val registeredAtMs = identity?.let { holder.segmentDemandRegisteredAtMs(request, it) } val nowMs = clock() val backoffRemainingMs = holder.session.demandBackoffRemainingMs - if (identity != null && registeredAtMs != null && deadline.isExpired( + val deadlineAtMs = identity?.let { demandIdentity -> + registeredAtMs?.let { registered -> + deadline.deadlineAtMs(demandIdentity, registered, nowMs, backoffRemainingMs) + } + } + if (identity != null && registeredAtMs != null && deadlineAtMs != null && nowMs >= deadlineAtMs && + SabrDemandAttemptFinisher.expireStalledDemand( + holder, + request, identity, - registeredAtMs, - nowMs, - backoffRemainingMs, - ) && - SabrDemandAttemptFinisher.expireStalledDemand(holder, request, identity, futureLiveRequest) + nowMs = nowMs, + expectedDelayMs = (deadlineAtMs - registeredAtMs).coerceAtLeast(0L), + ) ) { return true } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt index cbd6e9e9..3657d332 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePumpStep.kt @@ -18,7 +18,8 @@ internal suspend fun pumpLiveReadAhead( holder.setPlaybackState(SabrPlaybackState.REQUESTING) withLiveContinuationRequestShape(holder) { pump() } val cached = holder.cacheObservedLiveContinuation(onResolved) - holder.setPlaybackState(SabrPlaybackState.IDLE) + val waitingForLive = cached == 0 && holder.nextSegmentDemand()?.let(holder::isFutureLiveRequest) == true + holder.setPlaybackState(if (waitingForLive) SabrPlaybackState.WAITING_FOR_LIVE else SabrPlaybackState.IDLE) return cached > 0 && holder.observedLiveAheadMs() < targetAheadMs } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index e538b266..de439de9 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -106,23 +106,15 @@ class SabrSessionPumpLoop( holder.nextSegmentDemand()?.let { request -> val demandIdentity = holder.segmentDemandIdentity(request) ?: return true val wasFutureLiveRequest = holder.isFutureLiveRequest(request) - if (!holder.beginInFlightSegmentDemand(request, demandIdentity, wasFutureLiveRequest)) return true - try { - SabrPumpLogger.start(holder, "demand", request) - runtime.beginDemand(demandIdentity) - val result = pumpDemand(holder, localization, request, runtime) - return SabrDemandAttemptFinisher.finish( - holder, - request, - demandIdentity, - result, - runtime, - wasFutureLiveRequest, - onResolved = { onResolved(holder, it) }, - ) - } finally { - holder.finishInFlightSegmentDemand(demandIdentity) - } + return runSabrDemandAttempt( + holder = holder, + request = request, + identity = demandIdentity, + wasFutureLiveRequest = wasFutureLiveRequest, + runtime = runtime, + pump = { pumpDemand(holder, localization, request, runtime) }, + onResolved = { onResolved(holder, it) }, + ) } if (holder.livePlaybackSnapshot()?.active == true) { return pumpLiveReadAhead( From ea5c398878c286aa509f01a8aa484425afb4390b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 18:56:32 +0000 Subject: [PATCH 072/137] test: cover live SABR demand continuity --- .../SabrDemandWatchdogLifecycleTest.kt | 62 +++++++++++++++++++ .../services/SabrDownloadFrameWriterTest.kt | 17 +++-- .../services/SabrLiveSessionWarmupTest.kt | 20 ++++++ 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt index 0541b1e8..79d887d8 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt @@ -9,6 +9,7 @@ import kotlinx.coroutines.test.runCurrent import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.SabrSegmentRequest @@ -100,6 +101,67 @@ class SabrDemandWatchdogBackoffTest { } } + @Test + fun `future live demand waits past deadline in the same generation`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val job = launch { + SabrDemandWatchdog( + clock = { testScheduler.currentTime }, + intervalMs = 100L, + ).monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS) + runCurrent() + advanceTimeBy(LIVE_EDGE_POLL_MS) + runCurrent() + + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + job.cancel() + } + } + + @Test + fun `stalled live attempt is cancelled and requeued`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val identity = requireNotNull(holder.segmentDemandIdentity(request)) + assertTrue(holder.beginInFlightSegmentDemand(request, identity, futureLiveRequest = true)) + var cancelled = false + holder.registerInFlightDemandCancellation(identity) { cancelled = true } + val demand = requireNotNull(holder.inFlightSegmentDemand()) + + assertTrue( + SabrDemandAttemptFinisher.interruptStalledInFlightDemand( + holder, + demand, + nowMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + expectedDelayMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + ), + ) + + assertTrue(cancelled) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertNotEquals(identity, holder.segmentDemandIdentity(request)) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + } + } + private fun watchdog(clock: () -> Long): SabrDemandWatchdog = SabrDemandWatchdog( clock = clock, intervalMs = 100L, diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt index 4d183d57..30f75724 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDownloadFrameWriterTest.kt @@ -84,16 +84,15 @@ class SabrDemandWatchdogLifecycleTest { } @Test - fun `future live demand expires as recoverable`() = runTest { + fun `future live demand waits without terminal failure`() = runTest { withTracker { holder -> val request = SabrSegmentRequest.media(holder.videoFormat, 50) every { holder.session.isLive } returns true every { holder.session.streamState.isLive } returns true every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 49 holder.requestSegmentDemand(request, registeredAtMs = 0L) - var expired = false val job = launch { - expired = SabrDemandWatchdog( + SabrDemandWatchdog( clock = { testScheduler.currentTime }, intervalMs = 100L, ).monitor({ true }, holder) @@ -105,12 +104,12 @@ class SabrDemandWatchdogLifecycleTest { advanceTimeBy(LIVE_EDGE_POLL_MS) runCurrent() - assertTrue(job.isCompleted) - assertTrue(expired) - assertEquals( - "$SABR_RECOVERABLE_FAILURE_PREFIX SABR demand stalled for 299:50", - holder.terminalFailure(), - ) + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(0L, holder.activeGeneration()) + assertEquals(null, holder.terminalFailure()) + job.cancel() } } diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt index 70915d31..5ce991cf 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -4,10 +4,12 @@ import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import dev.typetype.server.sabr.SabrMediaHeader import dev.typetype.server.sabr.SabrMediaSegment +import dev.typetype.server.sabr.SabrSegmentRequest import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import dev.typetype.server.sabr.YoutubeSabrSession @@ -36,6 +38,24 @@ class SabrLivePumpStepTest { assertFalse(immediate) } + @Test + fun `live pump waits when the next demand is not published yet`() = runTest { + SabrSegmentDemandTracker.clearAll() + try { + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) + every { fixture.holder.session.isLive } returns true + every { fixture.holder.session.streamState.isLive } returns true + every { fixture.holder.session.streamState.getMaxSegment(fixture.holder.videoFormat) } returns 100 + fixture.holder.requestSegmentDemand(SabrSegmentRequest.media(fixture.holder.videoFormat, 101)) + + pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { 0 }) { _, _ -> } + + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, fixture.holder.playbackState()) + } finally { + SabrSegmentDemandTracker.clearAll() + } + } + private fun fixture(playerTimeMs: Long, observedEndMs: Long): Fixture { val audio = format(140, true) val video = format(299, false) From 9ff2e09419734f991948a8212b222b25be827c27 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 20:02:04 +0000 Subject: [PATCH 073/137] fix: target retried live SABR demand --- .../server/services/SabrPumpRuntime.kt | 3 ++ .../server/services/SabrSessionPumpLoop.kt | 9 +++- .../SabrPumpRuntimeTargetedShapeTest.kt | 43 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt index e1b22ca2..1ca188eb 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpRuntime.kt @@ -37,6 +37,9 @@ class SabrPumpRuntime(private val clock: () -> Long = System::currentTimeMillis) if (demandKey == requestKey) resetDemandRecovery() } + fun demandNeedsTargetedShape(requestKey: String): Boolean = + demandKey == requestKey && demandTrackReadvertised + fun demandRecoveryAction( requestKey: String, requestPerformed: Boolean, diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt index de439de9..676260f5 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionPumpLoop.kt @@ -112,7 +112,7 @@ class SabrSessionPumpLoop( identity = demandIdentity, wasFutureLiveRequest = wasFutureLiveRequest, runtime = runtime, - pump = { pumpDemand(holder, localization, request, runtime) }, + pump = { pumpDemand(holder, localization, request, demandIdentity, runtime) }, onResolved = { onResolved(holder, it) }, ) } @@ -166,6 +166,7 @@ class SabrSessionPumpLoop( holder: SabrSessionHolder, localization: Localization, request: SabrSegmentRequest, + identity: String, runtime: SabrPumpRuntime, ): YoutubeSabrSession.DemandResponseResult { val edgeMs = holder.session.streamState.getMinBufferedEndMs() @@ -179,6 +180,12 @@ class SabrSessionPumpLoop( pumpUntilCached(holder, localization, request, runtime) } } + if (runtime.demandNeedsTargetedShape(identity)) { + holder.setPlaybackState(SabrPlaybackState.REPOSITIONING) + return withTargetedRequestShape(holder, request, prepareSession = false) { + pumpUntilCached(holder, localization, request, runtime) + } + } holder.setPlaybackState(SabrPlaybackState.REQUESTING) return withLiveContinuationRequestShape(holder) { pumpUntilCached(holder, localization, request, runtime) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt new file mode 100644 index 00000000..e8045bea --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpRuntimeTargetedShapeTest.kt @@ -0,0 +1,43 @@ +package dev.typetype.server.services + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SabrPumpRuntimeTargetedShapeTest { + @Test + fun `unresolved demand switches to targeted requests after readvertisement`() { + val runtime = SabrPumpRuntime() + + assertFalse(runtime.demandNeedsTargetedShape("demand")) + runtime.beginDemand("demand") + assertFalse(runtime.demandNeedsTargetedShape("demand")) + + assertEquals( + SabrDemandRecoveryAction.READVERTISE_TRACK, + runtime.demandRecoveryAction("demand", requestPerformed = true, resolved = false), + ) + assertTrue(runtime.demandNeedsTargetedShape("demand")) + assertEquals( + SabrDemandRecoveryAction.WAIT, + runtime.demandRecoveryAction("demand", requestPerformed = true, resolved = false), + ) + assertTrue(runtime.demandNeedsTargetedShape("demand")) + + runtime.finishDemand("demand") + assertFalse(runtime.demandNeedsTargetedShape("demand")) + } + + @Test + fun `a new demand starts with continuation requests`() { + val runtime = SabrPumpRuntime() + + runtime.beginDemand("first") + runtime.demandRecoveryAction("first", requestPerformed = true, resolved = false) + runtime.beginDemand("second") + + assertFalse(runtime.demandNeedsTargetedShape("first")) + assertFalse(runtime.demandNeedsTargetedShape("second")) + } +} From afe3c5036a7c81fd4c4df4cd126d07ee8bb4a877 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Tue, 22 Sep 2026 21:01:26 +0000 Subject: [PATCH 074/137] fix: requeue recoverable live SABR demands --- .../services/SabrDemandAttemptFinisher.kt | 23 ++++++------ .../server/services/SabrDemandWatchdog.kt | 36 +++++++++++++++++-- .../SabrDemandWatchdogLifecycleTest.kt | 31 ++++++++++++++++ 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt index 4b3f1c8d..2958e1a0 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandAttemptFinisher.kt @@ -28,10 +28,11 @@ object SabrDemandAttemptFinisher { demand: SabrInFlightDemand, nowMs: Long, expectedDelayMs: Long, + reason: String, ): Boolean = synchronized(holder) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false - if (!demand.futureLiveRequest || holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false + if (holder.inFlightSegmentDemand()?.identity != demand.identity) return@synchronized false if (holder.session.getCachedSegment(demand.request) != null) return@synchronized false if (!holder.requeueSegmentDemand(demand.request, demand.identity, nowMs)) return@synchronized false holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) @@ -47,10 +48,10 @@ object SabrDemandAttemptFinisher { holder, demand.request, recoverable = true, - registeredAtMs = demand.registeredAtMs, - nowMs = nowMs, - expectedDelayMs = expectedDelayMs, - reason = "future_live_not_published", + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = reason, attempts = demand.attempts(), lastAttemptDurationMs = demand.lastAttemptDurationMs(), event = "demand_requeued", @@ -64,6 +65,7 @@ object SabrDemandAttemptFinisher { recoverable: Boolean = demand.futureLiveRequest, nowMs: Long = System.currentTimeMillis(), expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason: String = "terminal_deadline", ): Boolean = synchronized(holder) { val state = holder.playbackState() @@ -83,10 +85,10 @@ object SabrDemandAttemptFinisher { holder, demand.request, recoverable, - registeredAtMs = demand.registeredAtMs, - nowMs = nowMs, - expectedDelayMs = expectedDelayMs, - reason = "terminal_deadline", + registeredAtMs = demand.registeredAtMs, + nowMs = nowMs, + expectedDelayMs = expectedDelayMs, + reason = reason, attempts = demand.attempts(), lastAttemptDurationMs = demand.lastAttemptDurationMs(), ) @@ -101,6 +103,7 @@ object SabrDemandAttemptFinisher { recoverable: Boolean = false, nowMs: Long = System.currentTimeMillis(), expectedDelayMs: Long = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason: String = "terminal_deadline", ): Boolean = synchronized(holder) { val state = holder.playbackState() if (state == SabrPlaybackState.TERMINAL || state == SabrPlaybackState.NETWORK_FAILED) return@synchronized false @@ -113,7 +116,7 @@ object SabrDemandAttemptFinisher { registeredAtMs = holder.segmentDemandRegisteredAtMs(request, identity), nowMs = nowMs, expectedDelayMs = expectedDelayMs, - reason = "terminal_deadline", + reason = reason, ) fail(holder, request, identity, recoverable) } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt index cfaf27f5..62f3c641 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrDemandWatchdog.kt @@ -21,6 +21,7 @@ class SabrDemandWatchdog( inFlightDemand, recoverable = true, nowMs = nowMs, + reason = "outside_recoverable_window", ) ) { return true @@ -39,18 +40,27 @@ class SabrDemandWatchdog( backoffRemainingMs, ) if (nowMs >= deadlineAtMs) { - if (inFlightDemand.futureLiveRequest) { + val recoverableLiveDemand = + holder.livePlaybackSnapshot()?.active == true && + !holder.isLiveDemandOutsideRecoverableWindow(inFlightDemand.request) + if (recoverableLiveDemand) { SabrDemandAttemptFinisher.interruptStalledInFlightDemand( holder, inFlightDemand, nowMs = nowMs, expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + reason = if (inFlightDemand.futureLiveRequest) { + "future_live_not_published" + } else { + "network_slow" + }, ) } else if (SabrDemandAttemptFinisher.expireStalledInFlightDemand( holder, inFlightDemand, nowMs = nowMs, expectedDelayMs = (deadlineAtMs - inFlightDemand.registeredAtMs).coerceAtLeast(0L), + reason = "terminal_deadline", ) ) { return true @@ -69,7 +79,13 @@ class SabrDemandWatchdog( if (outsideLiveWindow) { val identity = holder.segmentDemandIdentity(request) if (identity != null && - SabrDemandAttemptFinisher.expireStalledDemand(holder, request, identity, recoverable = true) + SabrDemandAttemptFinisher.expireStalledDemand( + holder, + request, + identity, + recoverable = true, + reason = "outside_recoverable_window", + ) ) { return true } @@ -79,6 +95,21 @@ class SabrDemandWatchdog( delay(nextCheckDelayMs(holder.session.demandBackoffRemainingMs, futureLiveRequest)) continue } + val recoverableLiveRequest = + holder.livePlaybackSnapshot()?.active == true && !outsideLiveWindow + if (recoverableLiveRequest && holder.inFlightSegmentDemand() == null) { + val identity = holder.segmentDemandIdentity(request) + val requeued = synchronized(holder) { + holder.inFlightSegmentDemand() == null && + identity != null && + holder.requeueSegmentDemand(request, identity, clock()) + } + if (requeued) { + holder.setPlaybackState(SabrPlaybackState.WAITING_FOR_LIVE) + delay(nextCheckDelayMs(holder.session.demandBackoffRemainingMs, true)) + continue + } + } val identity = holder.segmentDemandIdentity(request) val registeredAtMs = identity?.let { holder.segmentDemandRegisteredAtMs(request, it) } val nowMs = clock() @@ -95,6 +126,7 @@ class SabrDemandWatchdog( identity, nowMs = nowMs, expectedDelayMs = (deadlineAtMs - registeredAtMs).coerceAtLeast(0L), + reason = "terminal_deadline", ) ) { return true diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt index 79d887d8..28556fb2 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrDemandWatchdogLifecycleTest.kt @@ -151,6 +151,7 @@ class SabrDemandWatchdogBackoffTest { demand, nowMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, expectedDelayMs = SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS, + reason = "future_live_not_published", ), ) @@ -162,6 +163,36 @@ class SabrDemandWatchdogBackoffTest { } } + @Test + fun `recoverable late live demand waits past deadline`() = runTest { + withTracker { holder -> + val request = SabrSegmentRequest.media(holder.videoFormat, 50) + every { holder.session.isLive } returns true + every { holder.session.liveHeadSequenceNumber } returns 52L + every { holder.session.streamState.isLive } returns true + every { holder.session.streamState.liveHeadSequenceNumber } returns 52L + every { holder.session.streamState.liveHeadTimeMs } returns 1_000L + every { holder.session.streamState.getMaxSegment(holder.videoFormat) } returns 52 + holder.requestSegmentDemand(request, registeredAtMs = 0L) + val job = launch { + SabrDemandWatchdog( + clock = { testScheduler.currentTime }, + intervalMs = 100L, + ).monitor({ true }, holder) + } + runCurrent() + + advanceTimeBy(SabrPumpPolicy.DEMAND_TARGET_DEADLINE_MS + LIVE_EDGE_POLL_MS) + runCurrent() + + assertFalse(job.isCompleted) + assertEquals(SabrPlaybackState.WAITING_FOR_LIVE, holder.playbackState()) + assertEquals("299:50", holder.pendingSegmentDemandSummary()) + assertEquals(null, holder.terminalFailure()) + job.cancel() + } + } + private fun watchdog(clock: () -> Long): SabrDemandWatchdog = SabrDemandWatchdog( clock = clock, intervalMs = 100L, From 576a0241777a1a0493771565029dcedf2062644b Mon Sep 17 00:00:00 2001 From: kapdon <94782486+kapdon@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:17:24 +0900 Subject: [PATCH 075/137] feat: paginate subscription group memberships (#86) * test: support isolated external PostgreSQL test databases * feat: query subscription membership pages in SQL * feat: expose paged membership and selection lookup endpoints * fix: optimize membership filters and Unicode search * fix: keep subscription page counts consistent during additions * perf: scale subscription membership reads --------- Co-authored-by: web-flow Co-authored-by: Priveetee <138765817+Priveetee@users.noreply.github.com> --- openapi.yaml | 2 + openapi/components/subscriptions.yaml | 13 ++ .../paths/subscription-membership-pages.yaml | 72 ++++++++ .../models/SubscriptionMembershipPage.kt | 13 ++ .../server/db/DatabaseIndexMigrations.kt | 2 + .../dev/typetype/server/TestDatabase.kt | 10 +- .../services/SubscriptionMutationLock.kt | 7 + .../SubscriptionMembershipPageRoutes.kt | 56 ++++++ .../server/routes/SubscriptionsRoutes.kt | 1 + .../SubscriptionMembershipPageRoutesTest.kt | 109 ++++++++++++ .../services/SubscriptionAvatarRepairer.kt | 18 +- .../services/SubscriptionGroupQueries.kt | 35 ++++ .../services/SubscriptionGroupsService.kt | 25 +-- .../services/SubscriptionMembershipFilter.kt | 10 ++ .../SubscriptionMembershipPageService.kt | 103 +++++++++++ .../server/services/SubscriptionsService.kt | 1 + .../SubscriptionMembershipConcurrencyTest.kt | 155 +++++++++++++++++ .../SubscriptionMembershipPageServiceTest.kt | 160 ++++++++++++++++++ .../server/SubscriptionMembershipScaleTest.kt | 78 +++++++++ 19 files changed, 843 insertions(+), 27 deletions(-) create mode 100644 openapi/paths/subscription-membership-pages.yaml create mode 100644 server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt create mode 100644 server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt diff --git a/openapi.yaml b/openapi.yaml index 46a836d0..3b2c81c4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -47,6 +47,8 @@ paths: /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } /subscriptions/group-memberships: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupMemberships } + /subscriptions/group-memberships/page: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Page } + /subscriptions/group-memberships/lookup: { $ref: ./openapi/paths/subscription-membership-pages.yaml#/Lookup } /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index d0af4c01..9117764b 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -18,6 +18,19 @@ SubscriptionGroupMembershipItem: type: array uniqueItems: true items: { type: string, format: uuid } +SubscriptionMembershipPage: + type: object + required: [items, total, totalSubscriptions, ungroupedCount, page, limit] + properties: + items: + type: array + maxItems: 100 + items: { $ref: '#/SubscriptionGroupMembershipItem' } + total: { type: integer, format: int64, minimum: 0, description: Number of subscriptions matching all filters. } + totalSubscriptions: { type: integer, format: int64, minimum: 0, description: Unfiltered account subscription count. } + ungroupedCount: { type: integer, format: int64, minimum: 0, description: Unfiltered account count with no memberships. } + page: { type: integer, minimum: 0, maximum: 1000000 } + limit: { type: integer, minimum: 1, maximum: 100 } SubscriptionCreateRequest: type: object required: [channelUrl, name, avatarUrl] diff --git a/openapi/paths/subscription-membership-pages.yaml b/openapi/paths/subscription-membership-pages.yaml new file mode 100644 index 00000000..18025f48 --- /dev/null +++ b/openapi/paths/subscription-membership-pages.yaml @@ -0,0 +1,72 @@ +Page: + get: + tags: [user-data] + summary: Read a bounded page of subscriptions with group memberships + description: | + Filtering, counting and pagination run in the database before channel rows and their + memberships are loaded. Rows are ordered by lowercase name and then canonical channel URL. + The zero-based page is an offset into current results, not a snapshot; changes between + requests may move rows. A page beyond the last returns empty items with the current total. + Group IDs and results are scoped to the authenticated profile. Unknown or inaccessible + groupId returns subscription_group_not_found. Omit groupId and ungrouped for all subscriptions. + parameters: + - name: page + in: query + schema: { type: integer, minimum: 0, maximum: 1000000, default: 0 } + - name: limit + in: query + schema: { type: integer, minimum: 1, maximum: 100, default: 20 } + - name: search + in: query + description: Case-insensitive literal substring of channel name or URL. Whitespace is trimmed; SQL wildcard characters are literal. + schema: { type: string, maxLength: 200 } + - name: groupId + in: query + schema: { type: string, format: uuid } + - name: ungrouped + in: query + description: Select channels in no group. Cannot be combined with groupId. + schema: { type: boolean, default: false } + - name: excluded + in: query + description: Select channels outside groupId, including channels in other groups. Requires groupId. + schema: { type: boolean, default: false } + responses: + '200': + description: Requested page and counts, including all memberships for each returned channel. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionMembershipPage } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +Lookup: + post: + tags: [user-data] + summary: Refresh memberships for explicitly selected subscriptions + description: | + Read-only lookup for selections retained across pages. Accepts one channelUrl or up to + 500 channelUrls, with a 2048-character URL limit and a 1 MiB request-body limit. + URLs are canonicalized and deduplicated. Returns only existing subscriptions owned by + the authenticated profile, with all their memberships; missing channels are omitted. + Results have no guaranteed order. This endpoint does not change subscriptions or groups. + requestBody: + required: true + content: + application/json: + schema: + oneOf: + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + - { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipBatchRequest } + responses: + '200': + description: Current memberships for the selected channels still subscribed to. + content: + application/json: + schema: + type: array + maxItems: 500 + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '413': { $ref: ../components/common.yaml#/JsonError } diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt new file mode 100644 index 00000000..d4968a27 --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/models/SubscriptionMembershipPage.kt @@ -0,0 +1,13 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionMembershipPage( + val items: List, + val total: Long, + val totalSubscriptions: Long, + val ungroupedCount: Long, + val page: Int, + val limit: Int, +) diff --git a/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt index 647fb241..2cf6a811 100644 --- a/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/DatabaseIndexMigrations.kt @@ -8,6 +8,8 @@ object DatabaseIndexMigrations { exec("CREATE INDEX IF NOT EXISTS idx_history_user_watched_id ON history (user_id, watched_at DESC, id DESC)") exec("CREATE INDEX IF NOT EXISTS idx_playlist_videos_user_playlist_position ON playlist_videos (user_id, playlist_id, position)") exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_user_subscribed_at ON subscriptions (user_id, subscribed_at DESC)") + exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_name_trgm ON subscriptions USING gin (lower(name) gin_trgm_ops)") + exec("CREATE INDEX IF NOT EXISTS idx_subscriptions_channel_url_trgm ON subscriptions USING gin (lower(channel_url) gin_trgm_ops)") exec("CREATE INDEX IF NOT EXISTS idx_favorites_user_favorited_at ON favorites (user_id, favorited_at DESC)") exec("CREATE INDEX IF NOT EXISTS idx_watch_later_user_added_at ON watch_later (user_id, added_at DESC)") exec("CREATE INDEX IF NOT EXISTS idx_search_history_user_searched_at ON search_history (user_id, searched_at DESC)") diff --git a/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt index 0cdf86d2..9624f526 100644 --- a/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt +++ b/server-db/src/testFixtures/kotlin/dev/typetype/server/TestDatabase.kt @@ -75,7 +75,14 @@ object TestDatabase { if (initialized) return synchronized(this) { if (initialized) return - val (baseUrl, user, password) = runCatching { + val externalUrl = System.getenv("TEST_DATABASE_URL")?.takeIf(String::isNotBlank) + val (baseUrl, user, password) = if (externalUrl != null) { + Triple( + externalUrl, + firstNonBlank(System.getenv("TEST_DATABASE_USER"), "typetype"), + firstNonBlank(System.getenv("TEST_DATABASE_PASSWORD"), "typetype"), + ) + } else runCatching { val c = container Triple(c.jdbcUrl, c.username, c.password) }.getOrElse { @@ -96,6 +103,7 @@ object TestDatabase { private fun ensureSchemaExists(baseUrl: String, user: String, password: String, schema: String): Unit { DriverManager.getConnection(baseUrl, user, password).use { connection -> connection.createStatement().use { statement -> + statement.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public") statement.execute("CREATE SCHEMA IF NOT EXISTS \"$schema\"") } } diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt index 50b3c3b3..cb39a255 100644 --- a/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionMutationLock.kt @@ -10,6 +10,13 @@ object SubscriptionMutationLock { ) } + fun acquireRead(userId: String) { + val userKey = userId.hashCode() and Int.MAX_VALUE + TransactionManager.current().exec( + "SELECT pg_advisory_xact_lock_shared($LOCK_NAMESPACE, $userKey)", + ) + } + // Precomputed PostgreSQL hashtext('subscriptions'). private const val LOCK_NAMESPACE = 720_815_616 } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt new file mode 100644 index 00000000..90cac507 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionMembershipPageRoutes.kt @@ -0,0 +1,56 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.get +import io.ktor.server.routing.post + +internal fun Route.subscriptionMembershipPageRoutes(auth: AuthService, groups: SubscriptionGroupsService) { + val service = SubscriptionMembershipPageService() + get("/subscriptions/group-memberships/page") { + call.withJwtAuth(auth) { userId -> + val filter = call.membershipFilter() ?: return@withJwtAuth call.respond( + HttpStatusCode.BadRequest, ErrorResponse("Invalid membership page filter", "subscription_group_invalid_filter"), + ) + val groupId = filter.groupId + if (groupId != null && !groups.exists(userId, groupId)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } + call.respond(service.getPage(userId, filter)) + } + } + post("/subscriptions/group-memberships/lookup") { + call.withJwtAuth(auth) { userId -> + val body = call.receiveMembershipBody() ?: return@withJwtAuth + val urls = when (val request = call.receiveMembershipChannels(body) ?: return@withJwtAuth) { + is MembershipChannels.Single -> listOf(request.channelUrl) + is MembershipChannels.Batch -> request.channelUrls + } + call.respond(service.lookup(userId, urls)) + } + } +} + +private fun ApplicationCall.membershipFilter(): SubscriptionMembershipFilter? { + val params = request.queryParameters + val page = params["page"]?.toIntOrNull() ?: if (params["page"] == null) 0 else return null + val limit = params["limit"]?.toIntOrNull() ?: if (params["limit"] == null) 20 else return null + val search = params["search"]?.trim().orEmpty() + val group = params["groupId"] + val ungrouped = params["ungrouped"]?.toBooleanStrictOrNull() + ?: if (params["ungrouped"] == null) false else return null + val excluded = params["excluded"]?.toBooleanStrictOrNull() + ?: if (params["excluded"] == null) false else return null + if (page !in 0..1_000_000 || limit !in 1..100 || search.length > 200) return null + if (group != null && group.isBlank() || group != null && ungrouped || excluded && group == null) return null + return SubscriptionMembershipFilter(page, limit, search, group, ungrouped, excluded) +} diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 1903cc99..b5355e9b 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -29,6 +29,7 @@ internal fun Route.subscriptionsRoutes( groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), pushNotificationService: PushNotificationService? = null, ) { + subscriptionMembershipPageRoutes(authService, groupsService) get("/subscriptions/group-memberships") { call.withJwtAuth(authService) { userId -> call.respond(subscriptionsService.getAllWithGroupMemberships(userId)) diff --git a/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt new file mode 100644 index 00000000..b9ab3243 --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageRoutesTest.kt @@ -0,0 +1,109 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.SubscriptionMembershipPage +import dev.typetype.server.routes.subscriptionsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionMembershipPageRoutesTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + private val path = "/subscriptions/group-memberships" + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit): Unit = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionsRoutes(subscriptions, AuthService.fixed(TEST_USER_ID), groupsService = groups) } + } + block() + } + + @Test + fun `page and lookup require authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("$path/page").status) + assertEquals(HttpStatusCode.Unauthorized, client.post("$path/lookup").status) + } + + @Test + fun `page validates limits filters and group ownership`() = withApp { + for (query in listOf("page=-1", "page=x", "page=1000001", "limit=0", "limit=101", "limit=x", + "groupId=", "excluded=true", "ungrouped=maybe", "excluded=maybe", "groupId=x&ungrouped=true", + "search=${"a".repeat(201)}")) { + assertEquals(HttpStatusCode.BadRequest, get("$path/page?$query").status, query) + } + val foreign = (groups.create("foreign", "Hidden") as SubscriptionGroupWriteResult.Success).group + for (id in listOf("missing", foreign.id)) { + val response = get("$path/page?groupId=$id") + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("subscription_group_not_found")) + } + } + + @Test + fun `page contract returns requested rows and accurate counts without changing legacy reads`() = withApp { + for (index in 0..2) subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/$index", "Name $index", "avatar")) + val response = get("$path/page?page=1&limit=2&search=NAME") + assertEquals(HttpStatusCode.OK, response.status) + val page = Json.decodeFromString(response.bodyAsText()) + assertEquals(1, page.page) + assertEquals(2, page.limit) + assertEquals(3L, page.total) + assertEquals(3L, page.totalSubscriptions) + assertEquals(3L, page.ungroupedCount) + assertEquals("Name 2", page.items.single().name) + assertTrue(get(path).bodyAsText().startsWith("[")) + } + + @Test + fun `lookup enforces bounded bodies and returns only current owned subscriptions`() = withApp { + subscriptions.add(TEST_USER_ID, SubscriptionItem("https://example.com/channel/one", "One", "avatar")) + val valid = lookup("""{"channelUrls":["https://example.com/channel/one","missing"]}""") + assertEquals(HttpStatusCode.OK, valid.status) + assertTrue(valid.bodyAsText().contains("\"name\":\"One\"")) + assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":[]}""").status) + assertEquals(HttpStatusCode.BadRequest, lookup("""{"channelUrls":["${"x".repeat(2049)}"]}""").status) + assertEquals(HttpStatusCode.BadRequest, lookup("{\"channelUrls\":[${List(501) { "\"x\"" }.joinToString(",")}]}").status) + assertEquals(HttpStatusCode.PayloadTooLarge, lookup("x".repeat(1024 * 1024 + 1)).status) + } + + private suspend fun ApplicationTestBuilder.get(url: String): io.ktor.client.statement.HttpResponse = client.get(url) { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private suspend fun ApplicationTestBuilder.lookup(body: String): io.ktor.client.statement.HttpResponse = client.post("$path/lookup") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + header(HttpHeaders.ContentType, "application/json") + setBody(body) + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt index 34c7a309..23731819 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt @@ -18,19 +18,29 @@ import org.jetbrains.exposed.v1.jdbc.update object SubscriptionAvatarRepairer { fun repair(userId: String, items: List): List { + val avatars = recoverableAvatars(userId, items) + avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } + return items.withAvatars(avatars) + } + + fun resolve(userId: String, items: List): List = + items.withAvatars(recoverableAvatars(userId, items)) + + private fun recoverableAvatars(userId: String, items: List): Map { val candidateUrls = items.filter { it.avatarUrl.isBlank() } .map { it.channelUrl } .distinct() - if (candidateUrls.isEmpty()) return items + if (candidateUrls.isEmpty()) return emptyMap() val avatars = knownAvatars(userId = userId, channelUrls = candidateUrls) .entries .take(MAX_AVATAR_REPAIR_PER_REQUEST) .associate { it.toPair() } - if (avatars.isEmpty()) return items - avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } - return items.map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } + return avatars } + private fun List.withAvatars(avatars: Map): List = + if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } + private fun knownAvatars(userId: String, channelUrls: List): Map { val avatars = linkedMapOf() historyAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt new file mode 100644 index 00000000..20aa8494 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupQueries.kt @@ -0,0 +1,35 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.models.SubscriptionGroupItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.count +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal object SubscriptionGroupQueries { + fun all(userId: String): List { + val count = SubscriptionGroupMembershipsTable.channelUrl.count() + val counts = SubscriptionGroupMembershipsTable.select(SubscriptionGroupMembershipsTable.groupId, count) + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .groupBy(SubscriptionGroupMembershipsTable.groupId) + .associate { it[SubscriptionGroupMembershipsTable.groupId] to it[count].toInt() } + return SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) + .map { row -> SubscriptionGroupItem( + id = row[SubscriptionGroupsTable.id], + name = row[SubscriptionGroupsTable.name], + channelCount = counts[row[SubscriptionGroupsTable.id]] ?: 0, + createdAt = row[SubscriptionGroupsTable.createdAt], + updatedAt = row[SubscriptionGroupsTable.updatedAt], + ) } + } + + fun exists(userId: String, groupId: String): Boolean = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.any() +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt index 5c92769b..a93e926c 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -5,7 +5,6 @@ import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.SubscriptionGroupItem -import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -21,18 +20,11 @@ import java.util.UUID class SubscriptionGroupsService { suspend fun getAll(userId: String): List = DatabaseFactory.query { - val counts = SubscriptionGroupMembershipsTable.selectAll() - .where { SubscriptionGroupMembershipsTable.userId eq userId } - .groupingBy { it[SubscriptionGroupMembershipsTable.groupId] } - .eachCount() - SubscriptionGroupsTable.selectAll() - .where { SubscriptionGroupsTable.userId eq userId } - .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) - .map { it.toItem(counts[it[SubscriptionGroupsTable.id]] ?: 0) } + SubscriptionGroupQueries.all(userId) } suspend fun exists(userId: String, groupId: String): Boolean = DatabaseFactory.query { - groupExists(userId, groupId) + SubscriptionGroupQueries.exists(userId, groupId) } suspend fun create(userId: String, rawName: String): SubscriptionGroupWriteResult { @@ -176,10 +168,7 @@ class SubscriptionGroupsService { .map { it[SubscriptionGroupMembershipsTable.channelUrl] } } - private fun groupExists(userId: String, groupId: String): Boolean = - SubscriptionGroupsTable.selectAll().where { - (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) - }.any() + private fun groupExists(userId: String, groupId: String): Boolean = SubscriptionGroupQueries.exists(userId, groupId) private fun nameExists(userId: String, normalizedName: String): Boolean = SubscriptionGroupsTable.selectAll().where { @@ -193,14 +182,6 @@ class SubscriptionGroupsService { (SubscriptionGroupMembershipsTable.groupId eq groupId) }.count().toInt() - private fun ResultRow.toItem(channelCount: Int): SubscriptionGroupItem = SubscriptionGroupItem( - id = this[SubscriptionGroupsTable.id], - name = this[SubscriptionGroupsTable.name], - channelCount = channelCount, - createdAt = this[SubscriptionGroupsTable.createdAt], - updatedAt = this[SubscriptionGroupsTable.updatedAt], - ) - private fun normalizeDisplayName(value: String): String? = value.trim().takeIf { it.length in 1..MAX_GROUP_NAME_LENGTH } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt new file mode 100644 index 00000000..61a24eab --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipFilter.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.services + +data class SubscriptionMembershipFilter( + val page: Int = 0, + val limit: Int = 20, + val search: String = "", + val groupId: String? = null, + val ungrouped: Boolean = false, + val excluded: Boolean = false, +) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt new file mode 100644 index 00000000..b8252c11 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionMembershipPageService.kt @@ -0,0 +1,103 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupMembershipItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.models.SubscriptionMembershipPage +import org.jetbrains.exposed.v1.core.Op +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.exists +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.like +import org.jetbrains.exposed.v1.core.lowerCase +import org.jetbrains.exposed.v1.core.notExists +import org.jetbrains.exposed.v1.core.or +import org.jetbrains.exposed.v1.core.stringParam +import org.jetbrains.exposed.v1.jdbc.Query +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.selectAll + +class SubscriptionMembershipPageService { + suspend fun getPage(userId: String, filter: SubscriptionMembershipFilter): SubscriptionMembershipPage = + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(userId) + val owned = SubscriptionsTable.userId eq userId + val ungrouped = notExists(matchingMemberships(userId)) + val matching = owned and filterCondition(userId, filter) + val total = SubscriptionsTable.selectAll().where { matching }.count() + val items = SubscriptionsTable.selectAll().where { matching } + .orderBy(SubscriptionsTable.name.lowerCase() to SortOrder.ASC, SubscriptionsTable.channelUrl to SortOrder.ASC) + .limit(filter.limit).offset(filter.page.toLong() * filter.limit) + .map { it.toItem() } + SubscriptionMembershipPage( + items = withMemberships(userId, items), + total = total, + totalSubscriptions = SubscriptionsTable.selectAll().where { owned }.count(), + ungroupedCount = SubscriptionsTable.selectAll().where { owned and ungrouped }.count(), + page = filter.page, + limit = filter.limit, + ) + } + + suspend fun lookup(userId: String, channelUrls: List): List = + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(userId) + val urls = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + val items = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl inList urls) + }.map { it.toItem() } + withMemberships(userId, items) + } + + private fun filterCondition(userId: String, filter: SubscriptionMembershipFilter): Op { + val memberships = matchingMemberships(userId, filter.groupId) + val membership = when { + filter.ungrouped || filter.excluded -> notExists(memberships) + filter.groupId != null -> exists(memberships) + else -> Op.TRUE + } + val search = filter.search.trim() + if (search.isEmpty()) return membership + val escaped = search.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + val pattern = stringParam("%$escaped%").lowerCase() + return membership and ((SubscriptionsTable.name.lowerCase() like pattern) or + (SubscriptionsTable.channelUrl.lowerCase() like pattern)) + } + + private fun matchingMemberships(userId: String, groupId: String? = null): Query = + SubscriptionGroupMembershipsTable.select(SubscriptionGroupMembershipsTable.channelUrl).where { + val matching = (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq SubscriptionsTable.channelUrl) + groupId?.let { matching and (SubscriptionGroupMembershipsTable.groupId eq it) } ?: matching + } + + private fun withMemberships(userId: String, items: List): List { + if (items.isEmpty()) return emptyList() + val urls = items.map { it.channelUrl } + val groups = SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl inList urls) + }.groupBy({ it[SubscriptionGroupMembershipsTable.channelUrl] }, { it[SubscriptionGroupMembershipsTable.groupId] }) + return SubscriptionAvatarRepairer.resolve(userId = userId, items = items).map { item -> + SubscriptionGroupMembershipItem( + channelUrl = item.channelUrl, + name = item.name, + avatarUrl = item.avatarUrl, + subscribedAt = item.subscribedAt, + groupIds = groups[item.channelUrl].orEmpty().sorted(), + ) + } + } + + private fun ResultRow.toItem(): SubscriptionItem = SubscriptionItem( + channelUrl = this[SubscriptionsTable.channelUrl], + name = this[SubscriptionsTable.name], + avatarUrl = this[SubscriptionsTable.avatarUrl], + subscribedAt = this[SubscriptionsTable.subscribedAt], + ) +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index b0bb5dd9..840e7d68 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -61,6 +61,7 @@ class SubscriptionsService { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) val now = System.currentTimeMillis() DatabaseFactory.query { + SubscriptionMutationLock.acquire(userId) SubscriptionsTable.insert { it[SubscriptionsTable.userId] = userId it[channelUrl] = canonicalUrl diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt new file mode 100644 index 00000000..ee8f52e8 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipConcurrencyTest.kt @@ -0,0 +1,155 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import dev.typetype.server.services.SubscriptionMutationLock +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class SubscriptionMembershipConcurrencyTest { + private val subscriptions = SubscriptionsService() + private val pages = SubscriptionMembershipPageService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `page reads and additions wait for the same account mutation`() = runTest { + val lockHeld = CompletableDeferred() + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + SubscriptionMutationLock.acquire(TEST_USER_ID) + val pid = TransactionManager.current().exec("SELECT pg_backend_pid()") { result -> + result.next() + result.getInt(1) + } + lockHeld.complete(requireNotNull(pid)) + check(releaseLock.await(10, TimeUnit.SECONDS)) + } + } + try { + val holderPid = lockHeld.await() + val addition = async(Dispatchers.IO) { subscriptions.add(TEST_USER_ID, subscription(0)) } + val read = async(Dispatchers.IO) { pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()) } + val otherAccount = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { subscriptions.add("other-account", subscription(0)) } + } + assertEquals(subscription(0).channelUrl, otherAccount?.channelUrl) + val bothWaited = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { + while (!addition.isCompleted && !read.isCompleted && waitingTransactions(holderPid) < 2) { + delay(10) + } + waitingTransactions(holderPid) >= 2 + } ?: false + } + assertTrue(bothWaited, "page reads and additions must wait for the same account-scoped lock") + releaseLock.countDown() + holder.await() + addition.await() + val page = read.await() + assertEquals(page.total, page.totalSubscriptions) + assertEquals(page.total, page.ungroupedCount) + assertEquals(page.total.toInt(), page.items.size) + assertEquals(1L, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).total) + } finally { + releaseLock.countDown() + } + } + + @Test + fun `same-account page reads share the lock while additions still wait`() = runTest { + val lockHeld = CompletableDeferred() + val releaseLock = CountDownLatch(1) + val holder = async(Dispatchers.IO) { + DatabaseFactory.query { + SubscriptionMutationLock.acquireRead(TEST_USER_ID) + val pid = TransactionManager.current().exec("SELECT pg_backend_pid()") { result -> + result.next() + result.getInt(1) + } + lockHeld.complete(requireNotNull(pid)) + check(releaseLock.await(10, TimeUnit.SECONDS)) + } + } + try { + val holderPid = lockHeld.await() + val pageRead = async(Dispatchers.IO) { pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()) } + val page = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { pageRead.await() } + } + assertEquals(0L, page?.total, "a concurrent page read must not wait for another reader") + + val addition = async(Dispatchers.IO) { subscriptions.add(TEST_USER_ID, subscription(0)) } + val additionWaited = withContext(Dispatchers.IO) { + withTimeoutOrNull(5_000L) { + while (!addition.isCompleted && waitingTransactions(holderPid) < 1) delay(10) + waitingTransactions(holderPid) >= 1 + } ?: false + } + assertTrue(additionWaited, "an addition must wait for active page readers") + releaseLock.countDown() + holder.await() + addition.await() + } finally { + releaseLock.countDown() + } + } + + @Test + fun `page rows and counts remain consistent during concurrent additions`() = runTest { + val start = CompletableDeferred() + val reader = async(Dispatchers.IO) { + start.await() + repeat(100) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 20)) + assertEquals(page.total, page.totalSubscriptions) + assertEquals(page.total, page.ungroupedCount) + assertEquals(minOf(page.total, 20L).toInt(), page.items.size) + } + } + val writer = async(Dispatchers.IO) { + start.await() + repeat(100) { subscriptions.add(TEST_USER_ID, subscription(it)) } + } + start.complete(Unit) + reader.await() + writer.await() + assertEquals(100L, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).total) + } + + private suspend fun waitingTransactions(holderPid: Int): Int = DatabaseFactory.query { + TransactionManager.current().exec( + "SELECT count(*) FROM pg_stat_activity WHERE $holderPid = ANY(pg_blocking_pids(pid))", + ) { result -> + result.next() + result.getInt(1) + } ?: 0 + } + + private fun subscription(index: Int): SubscriptionItem = + SubscriptionItem("https://example.com/channel/$index", "Channel $index", "avatar") +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt new file mode 100644 index 00000000..eab35b5c --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipPageServiceTest.kt @@ -0,0 +1,160 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.batchInsert +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionMembershipPageServiceTest { + private val pages = SubscriptionMembershipPageService() + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `large libraries return only the requested stable page with account totals`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.batchInsert(0..1000, shouldReturnGeneratedValues = false) { index -> + this[SubscriptionsTable.userId] = TEST_USER_ID + this[SubscriptionsTable.channelUrl] = url(index) + this[SubscriptionsTable.name] = "Channel ${index.toString().padStart(4, '0')}" + this[SubscriptionsTable.avatarUrl] = "avatar" + this[SubscriptionsTable.subscribedAt] = 1L + } + } + subscriptions.add("foreign", SubscriptionItem(url(0), "Foreign", "avatar")) + val first = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 7)) + val next = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(page = 1, limit = 7)) + assertEquals(1001L, first.total) + assertEquals(1001L, first.totalSubscriptions) + assertEquals(1001L, first.ungroupedCount) + assertEquals((0..6).map(::url), first.items.map { it.channelUrl }) + assertEquals((7..13).map(::url), next.items.map { it.channelUrl }) + assertTrue(pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(page = 999, limit = 7)).items.isEmpty()) + assertEquals(first, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(limit = 7))) + } + + @Test + fun `membership filters intersect literal search while returning all groups on each row`() = runTest { + for (index in 0..3) subscriptions.add(TEST_USER_ID, SubscriptionItem(url(index), "SAME $index", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(4), "100%_\\done", "avatar")) + val a = (groups.create(TEST_USER_ID, "A") as SubscriptionGroupWriteResult.Success).group + val b = (groups.create(TEST_USER_ID, "B") as SubscriptionGroupWriteResult.Success).group + groups.addSubscriptions(TEST_USER_ID, a.id, listOf(url(0), url(1))) + groups.addSubscriptions(TEST_USER_ID, b.id, listOf(url(1), url(2))) + val foreign = (groups.create("foreign", "Private") as SubscriptionGroupWriteResult.Success).group + subscriptions.add("foreign", SubscriptionItem(url(3), "Foreign channel", "avatar")) + groups.addSubscription("foreign", foreign.id, url(3)) + val inGroup = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(groupId = a.id, search = "same")) + assertEquals(listOf(url(0), url(1)), inGroup.items.map { it.channelUrl }) + assertEquals(listOf(a.id, b.id).sorted(), inGroup.items[1].groupIds) + assertEquals(2L, inGroup.total) + assertEquals(5L, inGroup.totalSubscriptions) + assertEquals(2L, inGroup.ungroupedCount) + assertEquals(listOf(url(2), url(3)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(groupId = a.id, excluded = true, search = " same ")).items.map { it.channelUrl }) + assertEquals(listOf(url(3)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(ungrouped = true, search = "SAME")).items.map { it.channelUrl }) + assertEquals(listOf(url(4)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = "%_\\")).items.map { it.channelUrl }) + assertEquals(listOf(url(2)), pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = url(2))).items.map { it.channelUrl }) + } + + @Test + fun `lookup refreshes off-page selections and omits deleted or foreign subscriptions`() = runTest { + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(1), "One", "avatar")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(2), "Two", "avatar")) + subscriptions.add("foreign", SubscriptionItem(url(3), "Foreign", "avatar")) + val group = (groups.create(TEST_USER_ID, "A") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, url(2)) + subscriptions.delete(TEST_USER_ID, url(1)) + val selected = pages.lookup(TEST_USER_ID, listOf(url(1), url(2), url(2), url(3))) + assertEquals(1, selected.size) + assertEquals(url(2), selected.single().channelUrl) + assertEquals(listOf(group.id), selected.single().groupIds) + assertEquals(1, groups.getAll(TEST_USER_ID).single().channelCount) + } + + @Test + fun `page and lookup resolve avatars without persisting repairs`() = runTest { + val item = subscriptions.add(TEST_USER_ID, SubscriptionItem(url(1), "One", "")) + val avatarUrl = "https://example.com/avatar.jpg" + DatabaseFactory.query { + HistoryTable.insert { + it[HistoryTable.id] = "membership-avatar" + it[HistoryTable.userId] = TEST_USER_ID + it[HistoryTable.url] = "https://www.youtube.com/watch?v=avatar-source" + it[HistoryTable.title] = "Video" + it[HistoryTable.thumbnail] = "thumbnail" + it[HistoryTable.channelName] = "One" + it[HistoryTable.channelUrl] = item.channelUrl + it[HistoryTable.channelAvatar] = avatarUrl + it[HistoryTable.duration] = 1L + it[HistoryTable.progress] = 0L + it[HistoryTable.watchedAt] = 1L + } + } + + assertEquals(avatarUrl, pages.lookup(TEST_USER_ID, listOf(item.channelUrl)).single().avatarUrl) + assertEquals(avatarUrl, pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter()).items.single().avatarUrl) + val storedAvatar = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq TEST_USER_ID } + .single()[SubscriptionsTable.avatarUrl] + } + assertEquals("", storedAvatar) + } + + @Test + fun `Unicode names match exact search without treating wildcards as patterns`() = runTest { + val names = listOf("École", "İstanbul", "ΟΣ", "İzmir_%\\'News") + for ((index, name) in names.withIndex()) { + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(index), name, "avatar")) + } + subscriptions.add(TEST_USER_ID, SubscriptionItem(url(99), "İzmirAXNews", "avatar")) + for ((index, name) in names.withIndex()) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(search = name)) + assertEquals(1L, page.total, name) + assertEquals(listOf(url(index)), page.items.map { it.channelUrl }, name) + } + assertEquals("École", pages.getPage(TEST_USER_ID, + SubscriptionMembershipFilter(search = "ÉCOLE")).items.single().name) + } + + @Test + fun `Unicode channel URLs use the same search normalization`() = runTest { + val item = subscriptions.add(TEST_USER_ID, + SubscriptionItem("https://example.com/İstanbul", "Ordinary channel", "avatar")) + for (search in listOf("İstanbul", "istanbul")) { + val page = pages.getPage(TEST_USER_ID, SubscriptionMembershipFilter(search = search)) + assertEquals(1L, page.total, search) + assertEquals(item.channelUrl, page.items.single().channelUrl) + } + } + + private fun url(index: Int): String = "https://www.youtube.com/channel/test${index.toString().padStart(4, '0')}" +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt new file mode 100644 index 00000000..55cf501a --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionMembershipScaleTest.kt @@ -0,0 +1,78 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionMembershipFilter +import dev.typetype.server.services.SubscriptionMembershipPageService +import kotlinx.coroutines.test.runTest +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import kotlin.time.Duration.Companion.seconds +import kotlin.time.measureTimedValue + +class SubscriptionMembershipScaleTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `pages stay responsive with 25000 channels and 112500 memberships`() = runTest { + seedLibrary() + val pages = SubscriptionMembershipPageService() + val filters = listOf( + SubscriptionMembershipFilter(limit = 7) to 25_000L, + SubscriptionMembershipFilter(limit = 7, ungrouped = true) to 2_500L, + SubscriptionMembershipFilter(limit = 7, groupId = "scale-1", excluded = true) to 2_500L, + SubscriptionMembershipFilter(limit = 7, groupId = "scale-1") to 22_500L, + SubscriptionMembershipFilter(limit = 7, search = "Channel 1") to 10_000L, + ) + for ((filter, total) in filters) { + val (page, elapsed) = measureTimedValue { pages.getPage(TEST_USER_ID, filter) } + assertTrue(elapsed < 3.seconds, "$filter page read took $elapsed") + assertEquals(total, page.total) + assertEquals(25_000L, page.totalSubscriptions) + assertEquals(2_500L, page.ungroupedCount) + assertEquals(7, page.items.size) + assertEquals(7, page.items.map { it.channelUrl }.distinct().size) + assertTrue(page.items.all { channel -> + when { + filter.ungrouped || filter.excluded -> channel.groupIds.isEmpty() + filter.groupId != null -> channel.groupIds.size == 5 + else -> true + } + }) + } + } + + private suspend fun seedLibrary(): Unit = DatabaseFactory.query { + // This membership set exceeds PostgreSQL's default NOT IN hash budget. + val transaction = TransactionManager.current() + transaction.exec(""" + INSERT INTO subscriptions (user_id, channel_url, name, avatar_url, subscribed_at) + SELECT '$TEST_USER_ID', 'https://www.youtube.com/channel/UC' || lpad(n::text, 22, '0'), + 'Channel ' || lpad(n::text, 5, '0'), 'avatar', 1 + FROM generate_series(1, 25000) n + """.trimIndent()) + transaction.exec(""" + INSERT INTO subscription_groups (id, user_id, name, normalized_name, created_at, updated_at) + SELECT 'scale-' || n, '$TEST_USER_ID', 'Group ' || n, 'group ' || n, 1, 1 + FROM generate_series(1, 5) n + """.trimIndent()) + transaction.exec(""" + INSERT INTO subscription_group_memberships (group_id, user_id, channel_url, added_at) + SELECT g.id, s.user_id, s.channel_url, 1 + FROM subscriptions s JOIN subscription_groups g ON g.user_id = s.user_id + WHERE s.user_id = '$TEST_USER_ID' AND right(s.channel_url, 1) <> '0' + """.trimIndent()) + transaction.exec("ANALYZE subscriptions") + transaction.exec("ANALYZE subscription_group_memberships") + } +} From 4a85d928356b91f80b0ecda9e7185f38dd2f6f9f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 09:43:06 +0000 Subject: [PATCH 076/137] perf: reduce live SABR startup latency --- .../routes/SabrPlaybackWindowBuilder.kt | 171 +++--------------- .../server/routes/SabrPlaybackWindowTiming.kt | 9 +- .../routes/SabrPlaybackWindowTrackBuilder.kt | 144 +++++++++++++++ .../server/services/SabrLivePlayback.kt | 4 +- .../services/SabrPlaybackSessionService.kt | 4 +- .../server/services/SabrPumpPolicy.kt | 2 +- 6 files changed, 184 insertions(+), 150 deletions(-) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt index 183a4a76..f31e90e7 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt @@ -1,23 +1,14 @@ package dev.typetype.server.routes -import dev.typetype.server.services.CachedSabrSegment import dev.typetype.server.services.SabrInitializationData import dev.typetype.server.services.SabrSessionHolder import dev.typetype.server.services.SabrSessionStore -import dev.typetype.server.services.coversPlaybackTime -import dev.typetype.server.services.findCachedPlaybackMediaAt -import dev.typetype.server.services.isAcceptableLiveFollowingSegment -import dev.typetype.server.services.failLivePlaybackDiscontinuity import dev.typetype.server.services.livePlaybackSnapshot -import dev.typetype.server.services.playbackContinuationSequence -import dev.typetype.server.services.playbackSegmentStartMs import dev.typetype.server.services.resolvePlaybackStartMs -import dev.typetype.server.sabr.SabrSegmentRequest -import dev.typetype.server.sabr.YoutubeSabrFormat - -private const val MAX_SEGMENTS_PER_TRACK = 12 internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessionStore) { + private val trackBuilder = SabrPlaybackWindowTrackBuilder(sabrSessionStore) + suspend fun build( holder: SabrSessionHolder, request: SabrPlaybackWindowRequest, @@ -28,25 +19,43 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi SabrInitializationData.ingestRemembered(holder.audioFormat, holder) if (!effectiveRequest.audioOnly) SabrInitializationData.ingestRemembered(holder.videoFormat, holder) if (effectiveRequest.audioOnly) return buildAudioOnly(holder, effectiveRequest, live?.toResponse()) - val video = buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, live?.active == true) + val activeLive = live?.active == true + var video = trackBuilder.buildTrack(holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, activeLive) val decodeStartMs = video.track.segments.firstOrNull()?.startMs ?: effectiveRequest.playerTimeMs - val audioStartMs = if (live?.active == true) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs - val audio = buildTrack( + val audioStartMs = if (activeLive) minOf(effectiveRequest.playerTimeMs, decodeStartMs) else effectiveRequest.playerTimeMs + var audio = trackBuilder.buildTrack( holder, holder.audioFormat, effectiveRequest, audioStartMs, - live?.active == true, + activeLive, ) - val playbackStartMs = resolvedPlaybackStartMs( + var playbackStartMs = resolvedPlaybackStartMs( effectiveRequest.playerTimeMs, - live?.active == true, + activeLive, video.track, audio.track, ) + val readyAheadMs = readyAheadMs(effectiveRequest, activeLive) + var requestedReadyEndMs = playbackStartMs + readyAheadMs + if (activeLive) { + if (!video.covers(requestedReadyEndMs) && video.blockedRequest == null) { + video = trackBuilder.buildTrack( + holder, holder.videoFormat, effectiveRequest, effectiveRequest.playerTimeMs, activeLive, + requestedReadyEndMs, + ) + } + if (!audio.covers(requestedReadyEndMs) && audio.blockedRequest == null) { + audio = trackBuilder.buildTrack( + holder, holder.audioFormat, effectiveRequest, audioStartMs, activeLive, requestedReadyEndMs, + ) + } + playbackStartMs = resolvedPlaybackStartMs( + effectiveRequest.playerTimeMs, activeLive, video.track, audio.track, + ) + requestedReadyEndMs = playbackStartMs + readyAheadMs + } val blocked = blockedTrack(audio, video) - val readyAheadMs = readyAheadMs(effectiveRequest, live?.active == true) - val requestedReadyEndMs = playbackStartMs + readyAheadMs return SabrPlaybackWindowBuildResult( response = SabrPlaybackWindowReadyResponse( sessionId = holder.sessionToken, @@ -71,7 +80,7 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi request: SabrPlaybackWindowRequest, live: SabrLivePlaybackResponse?, ): SabrPlaybackWindowBuildResult { - val audio = buildTrack(holder, holder.audioFormat, request, request.playerTimeMs, live?.active == true) + val audio = trackBuilder.buildTrack(holder, holder.audioFormat, request, request.playerTimeMs, live?.active == true) val playbackStartMs = resolvedPlaybackStartMs(request.playerTimeMs, live?.active == true, audio.track) val readyEndMs = playbackStartMs + readyAheadMs(request, live?.active == true) return SabrPlaybackWindowBuildResult( @@ -95,126 +104,4 @@ internal class SabrPlaybackWindowBuilder(private val sabrSessionStore: SabrSessi sequenceOf(video, audio) .filter { it.blockedRequest != null } .minByOrNull { it.coveredEndMs } - private suspend fun buildTrack( - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - request: SabrPlaybackWindowRequest, - requestedStartMs: Long, - activeLive: Boolean, - ): TrackBuildResult { - val continuesServedTrack = !activeLive || holder.lastServedSequence(format) != null - val targetMs = if (continuesServedTrack) request.bufferedEndFor(format, requestedStartMs) else requestedStartMs.coerceAtLeast(0L) - val goalStartMs = if (activeLive) targetMs else request.playerTimeMs.coerceAtLeast(0L) - var goalEndMs = maxOf(goalStartMs, request.playerTimeMs.coerceAtLeast(0L)) + - request.bufferGoalMs.coerceAtLeast(1L) - val segments = mutableListOf() - var blockedBy: String? = null - var blockedRequest: SabrSegmentRequest? = null - var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) - var coveredEndMs = targetMs - val endSequence = if (activeLive) 0 else holder.session.streamState.getEndSegment(format).toInt() - var atEnd = false - while (segments.size < MAX_SEGMENTS_PER_TRACK) { - if (endSequence > 0 && seq > endSequence) { - atEnd = true - break - } - val mediaRequest = SabrSegmentRequest.media(format, seq) - var segment = sabrSessionStore.cachedSegment(holder, mediaRequest) - val expectedStartMs = if (segments.isEmpty()) targetMs else coveredEndMs - if (segment == null || segments.isEmpty() && !segment.coversPlaybackTime(holder, format, targetMs)) { - val authoritative = sabrSessionStore.findCachedPlaybackMediaAt( - holder, format, expectedStartMs, seq, activeLive && segments.isEmpty(), - ) - if (authoritative != null && - (!activeLive || authoritative.isAcceptableLiveFollowingSegment(seq, targetMs, continuesServedTrack)) - ) { - seq = authoritative.sequence - holder.session.streamState.jumpBufferedTo(format, seq) - segment = authoritative - } - } - if (segment == null) { - val progressive = if (activeLive) null else segments.appendProgressiveWindowSegment( - holder, format, seq, expectedStartMs, - ) - if (progressive != null) { - seq = progressive.nextSequence - coveredEndMs = progressive.coveredEndMs - if (coveredEndMs >= goalEndMs) break - if (progressive.hasReadableMedia) continue - blockedRequest = SabrSegmentRequest.media(format, seq) - blockedBy = "${format.trackName()}:${format.itag}:$seq pending" - break - } - blockedBy = "${format.trackName()}:${format.itag}:$seq pending" - blockedRequest = mediaRequest - break - } - if (segments.isEmpty() && holder.failLivePlaybackDiscontinuity( - format, targetMs, segment, continuesServedTrack && request.bufferedRanges.any { it.itag == format.itag }, - ) - ) { - blockedBy = "${format.trackName()}:${format.itag}:$seq discontinuity" - break - } - if (!activeLive && segments.isEmpty() && segment.startMs > targetMs && seq > 1) { - seq = holder.previousPlaybackSequence(format, seq, segment, targetMs) - continue - } - val windowSegment = segment.toWindowSegment( - holder, - format, - sabrSessionStore.resolvePlaybackDurationMs(holder, format, segment), - ) - segments += windowSegment - if (activeLive && segments.size == 1) { - goalEndMs = maxOf(goalEndMs, windowSegment.startMs + request.bufferGoalMs.coerceAtLeast(1L)) - } - coveredEndMs = windowSegment.startMs + windowSegment.durationMs - if (endSequence > 0 && seq >= endSequence) { - atEnd = true - break - } - if (coveredEndMs >= goalEndMs) break - seq++ - } - if (blockedBy == null && coveredEndMs < goalEndMs && !atEnd) { - blockedBy = "${format.trackName()}:${format.itag}:$seq window capped" - } - return TrackBuildResult( - track = SabrPlaybackWindowTrack( - mime = format.mimeType.orEmpty(), - initUrl = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/init" + - "?generation=${holder.activeGeneration()}", - segments = segments, - ), - blockedBy = blockedBy, - blockedRequest = blockedRequest, - coveredEndMs = coveredEndMs, - atEnd = atEnd, - ) - } - private fun CachedSabrSegment.toWindowSegment( - holder: SabrSessionHolder, - format: YoutubeSabrFormat, - resolvedDurationMs: Long, - ): SabrPlaybackWindowSegment { - val startMs = startMs.takeIf { it >= 0L } - ?: holder.playbackSegmentStartMs(format, sequence) - return SabrPlaybackWindowSegment( - url = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/segment/$sequence?generation=${holder.activeGeneration()}", - startMs = startMs, - durationMs = resolvedDurationMs, - ) - } - private data class TrackBuildResult( - val track: SabrPlaybackWindowTrack, - val blockedBy: String?, - val blockedRequest: SabrSegmentRequest?, - val coveredEndMs: Long, - val atEnd: Boolean, - ) { - fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs - } } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt index a13efd6a..3f21c92f 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt @@ -35,7 +35,12 @@ internal fun readyAheadMs(request: SabrPlaybackWindowRequest, activeLive: Boolea } else { MIN_READY_AHEAD_MS } - return minOf(request.bufferGoalMs.coerceAtLeast(1L), minimum) + return minOf(liveWindowBufferGoalMs(request.bufferGoalMs, activeLive), minimum) +} + +internal fun liveWindowBufferGoalMs(requestedMs: Long, activeLive: Boolean): Long { + val requested = requestedMs.coerceAtLeast(1L) + return if (activeLive) minOf(requested, LIVE_STARTUP_READY_AHEAD_MS) else requested } internal fun YoutubeSabrFormat.trackName(): String = if (isAudio) "audio" else "video" @@ -78,4 +83,4 @@ internal fun SabrSessionHolder.previousPlaybackSequence( } private const val MIN_READY_AHEAD_MS = 1_000L -private const val LIVE_STARTUP_READY_AHEAD_MS = 8_000L +private const val LIVE_STARTUP_READY_AHEAD_MS = 2_000L diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt new file mode 100644 index 00000000..52b337d4 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -0,0 +1,144 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.CachedSabrSegment +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.coversPlaybackTime +import dev.typetype.server.services.failLivePlaybackDiscontinuity +import dev.typetype.server.services.findCachedPlaybackMediaAt +import dev.typetype.server.services.isAcceptableLiveFollowingSegment +import dev.typetype.server.services.playbackContinuationSequence +import dev.typetype.server.services.playbackSegmentStartMs +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat + +private const val MAX_SEGMENTS_PER_TRACK = 12 + +internal class SabrPlaybackWindowTrackBuilder(private val sabrSessionStore: SabrSessionStore) { + suspend fun buildTrack( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + request: SabrPlaybackWindowRequest, + requestedStartMs: Long, + activeLive: Boolean, + requiredEndMs: Long? = null, + ): TrackBuildResult { + val continuesServedTrack = !activeLive || holder.lastServedSequence(format) != null + val targetMs = if (continuesServedTrack) request.bufferedEndFor(format, requestedStartMs) else requestedStartMs.coerceAtLeast(0L) + val goalStartMs = if (activeLive) targetMs else request.playerTimeMs.coerceAtLeast(0L) + val liveBufferGoalMs = liveWindowBufferGoalMs(request.bufferGoalMs, activeLive) + var goalEndMs = maxOf(goalStartMs, request.playerTimeMs.coerceAtLeast(0L)) + liveBufferGoalMs + if (activeLive && requiredEndMs != null) goalEndMs = maxOf(goalEndMs, requiredEndMs) + val segments = mutableListOf() + var blockedBy: String? = null + var blockedRequest: SabrSegmentRequest? = null + var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) + var coveredEndMs = targetMs + val endSequence = if (activeLive) 0 else holder.session.streamState.getEndSegment(format).toInt() + var atEnd = false + while (segments.size < MAX_SEGMENTS_PER_TRACK) { + if (endSequence > 0 && seq > endSequence) { + atEnd = true + break + } + val mediaRequest = SabrSegmentRequest.media(format, seq) + var segment = sabrSessionStore.cachedSegment(holder, mediaRequest) + val expectedStartMs = if (segments.isEmpty()) targetMs else coveredEndMs + if (segment == null || segments.isEmpty() && !segment.coversPlaybackTime(holder, format, targetMs)) { + val authoritative = sabrSessionStore.findCachedPlaybackMediaAt( + holder, format, expectedStartMs, seq, activeLive && segments.isEmpty(), + ) + if (authoritative != null && + (!activeLive || authoritative.isAcceptableLiveFollowingSegment(seq, targetMs, continuesServedTrack)) + ) { + seq = authoritative.sequence + holder.session.streamState.jumpBufferedTo(format, seq) + segment = authoritative + } + } + if (segment == null) { + val progressive = if (activeLive) null else segments.appendProgressiveWindowSegment( + holder, format, seq, expectedStartMs, + ) + if (progressive != null) { + seq = progressive.nextSequence + coveredEndMs = progressive.coveredEndMs + if (coveredEndMs >= goalEndMs) break + if (progressive.hasReadableMedia) continue + blockedRequest = SabrSegmentRequest.media(format, seq) + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + break + } + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + blockedRequest = mediaRequest + break + } + if (segments.isEmpty() && holder.failLivePlaybackDiscontinuity( + format, targetMs, segment, continuesServedTrack && request.bufferedRanges.any { it.itag == format.itag }, + ) + ) { + blockedBy = "${format.trackName()}:${format.itag}:$seq discontinuity" + break + } + if (!activeLive && segments.isEmpty() && segment.startMs > targetMs && seq > 1) { + seq = holder.previousPlaybackSequence(format, seq, segment, targetMs) + continue + } + val windowSegment = segment.toWindowSegment( + holder, + format, + sabrSessionStore.resolvePlaybackDurationMs(holder, format, segment), + ) + segments += windowSegment + if (activeLive && segments.size == 1) { + goalEndMs = maxOf(goalEndMs, windowSegment.startMs + liveBufferGoalMs) + } + coveredEndMs = windowSegment.startMs + windowSegment.durationMs + if (endSequence > 0 && seq >= endSequence) { + atEnd = true + break + } + if (coveredEndMs >= goalEndMs) break + seq++ + } + if (blockedBy == null && coveredEndMs < goalEndMs && !atEnd) { + blockedBy = "${format.trackName()}:${format.itag}:$seq window capped" + } + return TrackBuildResult( + track = SabrPlaybackWindowTrack( + mime = format.mimeType.orEmpty(), + initUrl = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/init" + + "?generation=${holder.activeGeneration()}", + segments = segments, + ), + blockedBy = blockedBy, + blockedRequest = blockedRequest, + coveredEndMs = coveredEndMs, + atEnd = atEnd, + ) + } + + private fun CachedSabrSegment.toWindowSegment( + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + resolvedDurationMs: Long, + ): SabrPlaybackWindowSegment { + val startMs = startMs.takeIf { it >= 0L } + ?: holder.playbackSegmentStartMs(format, sequence) + return SabrPlaybackWindowSegment( + url = "${SabrPlaybackPaths.mediaBasePath(holder.sessionToken)}/${format.itag}/segment/$sequence?generation=${holder.activeGeneration()}", + startMs = startMs, + durationMs = resolvedDurationMs, + ) + } +} + +internal data class TrackBuildResult( + val track: SabrPlaybackWindowTrack, + val blockedBy: String?, + val blockedRequest: SabrSegmentRequest?, + val coveredEndMs: Long, + val atEnd: Boolean, +) { + fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs +} diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt index 517085e3..7dd09fb4 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrLivePlayback.kt @@ -129,8 +129,8 @@ private fun dev.typetype.server.sabr.YoutubeSabrStreamState.observedEndMs( internal const val LIVE_EDGE_POLL_MS = 2_000L internal const val DEFAULT_PLAYBACK_RETRY_MS = 500L -private const val LIVE_TARGET_LATENCY_MS = 20_000L +private const val LIVE_TARGET_LATENCY_MS = 2_000L private const val LIVE_EDGE_TOLERANCE_MS = 15_000L -private const val LIVE_HISTORICAL_REQUEST_TOLERANCE_MS = LIVE_TARGET_LATENCY_MS + LIVE_EDGE_TOLERANCE_MS +private const val LIVE_HISTORICAL_REQUEST_TOLERANCE_MS = 35_000L private const val LIVE_DVR_WINDOW_MS = 12L * 60L * 60L * 1_000L internal const val LIVE_FUTURE_SEGMENT_TOLERANCE = 2 diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt index 93e5f21e..160775e5 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackSessionService.kt @@ -63,14 +63,13 @@ class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { ) } - private suspend fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { + private fun prepareLive(holder: SabrSessionHolder, startTimeMs: Long, audioOnly: Boolean) { holder.setActiveTracks(videoActive = !audioOnly, audioActive = true) holder.session.streamState.setSelectVideoFormatBeforeAudio(!audioOnly) if (startTimeMs == 0L) { holder.session.streamState.setPlayerTimeMs(OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS) holder.session.streamState.setWriteTopLevelPlayerTimeMs(false) } - sessionStore.ensureWarmed(holder, LIVE_INITIAL_PUMPS) } suspend fun seek( @@ -159,7 +158,6 @@ class SabrPlaybackSessionService(private val sessionStore: SabrSessionStore) { private companion object { const val PREPARING = "preparing" const val INITIALIZATION_PRELOAD_TIMEOUT_MS = 6_000L - const val LIVE_INITIAL_PUMPS = 8 const val OFFICIAL_LIVE_EDGE_PLAYER_TIME_MS = 9_007_199_254_740_991L } } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt index 83264102..34685d52 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPumpPolicy.kt @@ -12,7 +12,7 @@ object SabrPumpPolicy { const val SEEK_MODE_MS = 8_000L const val MIN_SERVER_READAHEAD_CUSHION_MS = 3_000L const val SERVER_AHEAD_MARGIN_MS = 16_000L - const val LIVE_READAHEAD_CUSHION_MS = 90_000L + const val LIVE_READAHEAD_CUSHION_MS = 2_000L const val DEMAND_TARGET_DEADLINE_MS = 15_000L const val MAX_DEMAND_BACKOFF_EXTENSION_MS = 30_000L const val MAX_PROTECTED_NO_MEDIA_RESPONSES = 5 From ac4bbdbdc67c6367444860d201ede18d422a4603 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 09:43:14 +0000 Subject: [PATCH 077/137] test: cover low latency live SABR startup --- .../server/routes/SabrLiveGapWindowTest.kt | 7 +- .../routes/SabrLivePlaybackOverlapTest.kt | 8 +- .../routes/SabrLivePlaybackStartupTest.kt | 96 +++++++++++++++++++ .../SabrLivePlaybackWindowBuilderTest.kt | 53 ++-------- .../server/services/SabrLivePlaybackTest.kt | 26 +++-- .../services/SabrLiveSessionWarmupTest.kt | 4 +- .../services/SabrMissingDemandRecoveryTest.kt | 16 ++-- .../server/services/SabrPumpPolicyTest.kt | 4 +- .../SabrUnauthorizedResponseRecoveryTest.kt | 5 +- 9 files changed, 136 insertions(+), 83 deletions(-) create mode 100644 server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt index 00e21bb7..d3152964 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLiveGapWindowTest.kt @@ -22,7 +22,7 @@ import java.time.Instant class SabrLiveGapWindowTest { @Test - fun `small live gap extends the current playback window`() = runTest { + fun `small live gap requests only the first segment covering the low latency runway`() = runTest { val audio = format(140, isAudio = true) val video = format(299, isAudio = false) val session = mockk(relaxed = true) @@ -58,10 +58,7 @@ class SabrLiveGapWindowTest { assertTrue(result.isReady) assertEquals(480_000L, result.response.startTimeMs) assertEquals( - listOf( - "/api/sabr/playback/session/299/segment/93?generation=0", - "/api/sabr/playback/session/299/segment/94?generation=0", - ), + listOf("/api/sabr/playback/session/299/segment/93?generation=0"), requireNotNull(result.response.video).segments.map { it.url }, ) assertNull(holder.terminalFailure()) diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt index 5d9d1d6b..e6af9561 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackOverlapTest.kt @@ -20,7 +20,7 @@ import java.time.Instant class SabrLivePlaybackOverlapTest { @Test - fun `audio overlap still covers the ready horizon after player time`() = runTest { + fun `audio overlap covers the minimal live ready horizon after player time`() = runTest { val audio = format(140, true) val video = format(137, false) val session = mockk(relaxed = true) @@ -54,8 +54,10 @@ class SabrLivePlaybackOverlapTest { ) assertTrue(result.isReady) - assertEquals(4, result.response.audio.segments.size) - assertTrue(result.response.audio.segments.last().let { it.startMs + it.durationMs } >= 176_925_494L) + assertTrue(result.response.audio.segments.size <= 3) + val endMs = result.response.audio.segments.last().let { it.startMs + it.durationMs } + assertTrue(endMs >= 176_919_494L) + assertTrue(endMs <= 176_923_494L) } private fun holder( diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt new file mode 100644 index 00000000..155a5a4d --- /dev/null +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackStartupTest.kt @@ -0,0 +1,96 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.CachedSabrSegment +import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.services.SabrSessionKey +import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.sabr.SabrSegmentRequest +import dev.typetype.server.sabr.YoutubeSabrFormat +import dev.typetype.server.sabr.YoutubeSabrInfo +import dev.typetype.server.sabr.YoutubeSabrSession +import dev.typetype.server.sabr.YoutubeSabrStreamState +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.time.Instant + +class SabrLivePlaybackStartupTest { + @Test + fun `active live startup responds after the first media pair`() = runTest { + val audio = format(itag = 140, isAudio = true) + val video = format(itag = 299, isAudio = false) + val session = mockk(relaxed = true) + val streamState = mockk(relaxed = true) + every { session.streamState } returns streamState + every { session.isLive } returns true + every { streamState.isLive } returns true + every { streamState.liveHeadTimeMs } returns 120_000L + every { streamState.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 50 + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } answers { + val request = secondArg() + if (request.sequenceNumber in 50..52) { + cached(request.format.itag, request.sequenceNumber, 100_000L + (request.sequenceNumber - 50) * 2_000L) + } else { + null + } + } + + val builder = SabrPlaybackWindowBuilder(store) + val request = SabrPlaybackWindowRequest(0L, 100_000L, 299, 140, bufferGoalMs = 8_000L) + val startup = builder.build(holder, request) + val continuation = builder.build( + holder, + request.copy( + bufferedRanges = listOf( + SabrPlaybackBufferedRange(140, 0L, 100_000L), + SabrPlaybackBufferedRange(299, 0L, 100_000L), + ), + ), + ) + + assertTrue(startup.isReady) + assertTrue(continuation.isReady) + assertEquals(1, startup.response.audio.segments.size) + assertEquals(1, requireNotNull(startup.response.video).segments.size) + assertEquals(1, continuation.response.audio.segments.size) + assertEquals(1, requireNotNull(continuation.response.video).segments.size) + } + + private fun holder( + session: YoutubeSabrSession, + audio: YoutubeSabrFormat, + video: YoutubeSabrFormat, + ) = SabrSessionHolder( + session = session, + info = mockk(), + audioFormat = audio, + videoFormat = video, + sessionToken = "session", + key = SabrSessionKey("video", "user", audio.itag, null, video.itag, 0L), + lastRequestAt = Instant.EPOCH, + ) + + private fun format(itag: Int, isAudio: Boolean): YoutubeSabrFormat = mockk { + every { this@mockk.itag } returns itag + every { this@mockk.isAudio } returns isAudio + every { mimeType } returns if (isAudio) "audio/mp4" else "video/mp4" + every { approxDurationMs } returns 900_000L + } + + private fun cached(itag: Int, sequence: Int, startMs: Long): CachedSabrSegment = CachedSabrSegment( + itag = itag, + sequence = sequence, + init = false, + startMs = startMs, + durationMs = 2_000L, + mimeType = if (itag == 140) "audio/mp4" else "video/mp4", + bytesBase64 = "AA==", + byteLength = 1, + ) +} diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt index 21613725..e59955df 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrLivePlaybackWindowBuilderTest.kt @@ -64,7 +64,7 @@ class SabrLivePlaybackWindowBuilderTest { } @Test - fun `live window requests every track missing from the target window`() = runTest { + fun `live window only requires the first segment pair even for a large requested buffer`() = runTest { val audio = format(itag = 140, isAudio = true) val video = format(itag = 248, isAudio = false) val session = mockk(relaxed = true) @@ -93,8 +93,9 @@ class SabrLivePlaybackWindowBuilderTest { SabrPlaybackWindowRequest(0L, 100_000L, 248, 140, bufferGoalMs = 30_000L), ) - assertEquals(listOf(140, 248), result.blockedRequests.map { it.format.itag }.sorted()) - assertEquals(listOf(101, 106), result.blockedRequests.map { it.sequenceNumber }) + assertTrue(result.isReady) + assertEquals(1, result.response.audio.segments.size) + assertEquals(1, requireNotNull(result.response.video).segments.size) } @Test @@ -130,46 +131,6 @@ class SabrLivePlaybackWindowBuilderTest { assertEquals(102_010L, result.response.audio.segments.single().startMs) } - @Test - fun `active live startup waits for the full media cushion`() = runTest { - val audio = format(itag = 140, isAudio = true) - val video = format(itag = 299, isAudio = false) - val session = mockk(relaxed = true) - val streamState = mockk(relaxed = true) - every { session.streamState } returns streamState - every { session.isLive } returns true - every { streamState.isLive } returns true - every { streamState.liveHeadTimeMs } returns 120_000L - every { streamState.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 50 - val holder = holder(session, audio, video) - val store = mockk() - coEvery { store.cachedSegment(holder, any()) } answers { - val request = secondArg() - if (request.sequenceNumber in 50..52) { - cached(request.format.itag, request.sequenceNumber, 100_000L + (request.sequenceNumber - 50) * 2_000L, 2_000L) - } else { - null - } - } - - val builder = SabrPlaybackWindowBuilder(store) - val request = SabrPlaybackWindowRequest(0L, 100_000L, 299, 140, bufferGoalMs = 8_000L) - val startup = builder.build(holder, request) - val continuation = builder.build( - holder, - request.copy( - bufferedRanges = listOf( - SabrPlaybackBufferedRange(140, 0L, 100_000L), - SabrPlaybackBufferedRange(299, 0L, 100_000L), - ), - ), - ) - - assertFalse(startup.isReady) - assertTrue(continuation.isReady) - assertEquals(listOf(53, 53), continuation.blockedRequests.map { it.sequenceNumber }) - } - @Test fun `active live startup derives timing from adjacent cached segments`() = runTest { val audio = format(itag = 140, isAudio = true) @@ -206,12 +167,12 @@ class SabrLivePlaybackWindowBuilderTest { ) assertTrue(result.isReady) assertEquals(100_000L, result.response.startTimeMs) - assertEquals(4, result.response.audio.segments.size) - assertEquals(List(4) { 2_000L }, result.response.audio.segments.map { it.durationMs }) + assertEquals(1, result.response.audio.segments.size) + assertEquals(List(1) { 2_000L }, result.response.audio.segments.map { it.durationMs }) } @Test - fun `asymmetric live buffers keep requesting the shorter track`() = runTest { + fun `asymmetric live buffers request the shorter track at the shared playback start`() = runTest { val audio = format(itag = 140, isAudio = true) val video = format(itag = 299, isAudio = false) val session = mockk(relaxed = true) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt index e1b3d56c..7134c8ad 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLivePlaybackTest.kt @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger class SabrLivePlaybackSessionServiceTest { @Test - fun `live start uses the warmed media pair when it is closer to the head`() { + fun `live start uses the target when available media is behind it`() { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val holder = holder(audio, video) @@ -36,11 +36,11 @@ class SabrLivePlaybackSessionServiceTest { holder.observeMediaSegment(mediaSegment(audio.itag, 995_002L)) holder.observeMediaSegment(mediaSegment(video.itag, 995_000L)) - assertEquals(995_002L, holder.resolvePlaybackStartMs(0L)) + assertEquals(1_003_000L, holder.resolvePlaybackStartMs(0L)) } @Test - fun `live prepare warms metadata and starts behind the live head`() = runTest { + fun `live prepare starts near the live head without waiting for warmup`() = runTest { val audio = format(140, isAudio = true) val video = format(137, isAudio = false) val info = mockk() @@ -60,10 +60,10 @@ class SabrLivePlaybackSessionServiceTest { every { state.getBufferedEndMs(audio) } returns 1_000_000L every { state.getBufferedEndMs(video) } returns 1_002_000L every { state.getMinBufferedEndMs() } returns 1_000_000L - every { state.getSegmentNumberAtOrAfterTimeMs(video, 985_000L) } returns 198 - every { state.getSegmentNumberAtOrAfterTimeMs(audio, 985_000L) } returns 99 - every { state.getSegmentStartMs(video, 198) } returns 990_000L - every { state.getSegmentStartMs(audio, 99) } returns 990_000L + every { state.getSegmentNumberAtOrAfterTimeMs(video, 1_003_000L) } returns 200 + every { state.getSegmentNumberAtOrAfterTimeMs(audio, 1_003_000L) } returns 100 + every { state.getSegmentStartMs(video, 200) } returns 1_002_000L + every { state.getSegmentStartMs(audio, 100) } returns 1_000_000L val store = mockk() every { store.getOrCreate( @@ -80,17 +80,16 @@ class SabrLivePlaybackSessionServiceTest { 0L, ) } returns holder - coEvery { store.ensureWarmed(holder, 8) } returns Unit every { store.startPump(holder) } returns Unit val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) - assertEquals(985_000L, result.startTimeMs) - assertEquals(985_000L, holder.playerTimeMs()) + assertEquals(1_003_000L, result.startTimeMs) + assertEquals(1_003_000L, holder.playerTimeMs()) assertTrue(holder.expectsLive()) verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 1) { state.setWriteTopLevelPlayerTimeMs(false) } - coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } coVerify(exactly = 0) { store.fetchInitializationData(any(), any()) } verify(exactly = 1) { store.startPump(holder) } } @@ -161,7 +160,7 @@ class SabrLivePlaybackSessionServiceTest { } @Test - fun `live format change starts the replacement session from warmed track boundaries`() = runTest { + fun `live format change starts the replacement session without synchronous warmup`() = runTest { val audio = format(140, isAudio = true) val source = holder(audio, format(137, isAudio = false), initialGeneration = 4L) val video = format(248, isAudio = false) @@ -201,7 +200,6 @@ class SabrLivePlaybackSessionServiceTest { 5L, ) } returns replacement - coEvery { store.ensureWarmed(replacement, 8) } returns Unit every { store.startPump(replacement) } returns Unit val result = SabrPlaybackSessionService(store).seek(source, prepared, audio, video, 995_000L) @@ -217,7 +215,7 @@ class SabrLivePlaybackSessionServiceTest { assertFalse(replacement.mediaRequestsAt(995_000L).any { it.format.itag == video.itag }) verify(exactly = 0) { replacementState.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 0) { replacementState.setWriteTopLevelPlayerTimeMs(false) } - coVerify(exactly = 1) { store.ensureWarmed(replacement, 8) } + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } verify(exactly = 1) { store.startPump(replacement) } } diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt index 5ce991cf..b8ad0f09 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrLiveSessionWarmupTest.kt @@ -18,7 +18,7 @@ import java.time.Instant class SabrLivePumpStepTest { @Test fun `live pump fills below read ahead cushion`() = runTest { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 101_000L) var pumps = 0 val immediate = pumpLiveReadAhead(fixture.holder, SabrPumpRuntime(), { pumps++; 0 }) { _, _ -> } @@ -42,7 +42,7 @@ class SabrLivePumpStepTest { fun `live pump waits when the next demand is not published yet`() = runTest { SabrSegmentDemandTracker.clearAll() try { - val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 103_000L) + val fixture = fixture(playerTimeMs = 100_000L, observedEndMs = 101_000L) every { fixture.holder.session.isLive } returns true every { fixture.holder.session.streamState.isLive } returns true every { fixture.holder.session.streamState.getMaxSegment(fixture.holder.videoFormat) } returns 100 diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt index 53ae9e5b..a48e6860 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrMissingDemandRecoveryTest.kt @@ -18,7 +18,7 @@ import java.time.Instant class SabrLiveSessionWarmupTest { @Test - fun `warmup continues when the first media pair is too close to the live head`() = runTest { + fun `warmup accepts the first media pair at the low latency live edge`() = runTest { val audio = format(140, audio = true, "audio/mp4") val video = format(299, audio = false, "video/mp4") val streamState = mockk(relaxed = true) @@ -49,10 +49,10 @@ class SabrLiveSessionWarmupTest { SabrSessionPump(SabrSegmentCache()).ensureWarmed(holder, maxPumps = 8) - assertEquals(2, pumps) - assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(audio)) - assertEquals(1_980_000L, holder.earliestObservedMediaStartMs(video)) - assertEquals(1_980_000L, holder.resolvePlaybackStartMs(0L)) + assertEquals(1, pumps) + assertEquals(2_000_000L, holder.earliestObservedMediaStartMs(audio)) + assertEquals(2_000_000L, holder.earliestObservedMediaStartMs(video)) + assertEquals(2_000_000L, holder.resolvePlaybackStartMs(0L)) } @Test @@ -120,11 +120,11 @@ class SabrLiveSessionWarmupTest { assertEquals(TARGET_SEQUENCE, holder.observedMediaSegment(video)?.header?.sequenceNumber) assertArrayEquals(audioInit, holder.liveInitialization(audio)) assertArrayEquals(videoInit, holder.liveInitialization(video)) - assertEquals(liveHeadTimeMs(pumps) - 20_000L, holder.resolvePlaybackStartMs(0L)) + assertEquals(liveHeadTimeMs(pumps) - 2_000L, holder.resolvePlaybackStartMs(0L)) val targetedRanges = rangeOverrides.filterNotNull().map { ranges -> ranges.map(SabrBufferedRange::summarize) } val expectedRanges = listOf( - "itag=140:seq=1-3319:time=0+6640000:timescale=1000", - "itag=299:seq=1-3319:time=0+6640000:timescale=1000", + "itag=140:seq=1-3328:time=0+6658000:timescale=1000", + "itag=299:seq=1-3328:time=0+6658000:timescale=1000", ) assertEquals(2, targetedRanges.size) assertEquals(expectedRanges, targetedRanges.first()) diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt index d2d08b4b..0a32c144 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrPumpPolicyTest.kt @@ -25,7 +25,7 @@ import java.time.Instant @OptIn(ExperimentalCoroutinesApi::class) class SabrPumpLauncherTest { @Test - fun `active live playback keeps a bounded larger read ahead cushion`() { + fun `active live playback keeps read ahead to the startup target`() { val audio = format(140, isAudio = true) val video = format(136, isAudio = false) val state = mockk(relaxed = true) @@ -48,7 +48,7 @@ class SabrPumpLauncherTest { val runtime = SabrPumpRuntime { now } now = 30_000L - assertEquals(90_000L, runtime.targetReadaheadCushionMs(holder)) + assertEquals(2_000L, runtime.targetReadaheadCushionMs(holder)) } @Test diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt index d7e90ce1..400bb8cd 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrUnauthorizedResponseRecoveryTest.kt @@ -59,7 +59,6 @@ class SabrTransitioningLivePlaybackTest { } returns holder coEvery { store.fetchInitializationData(holder, video) } returns null coEvery { store.fetchInitializationData(holder, audio) } returns null - coEvery { store.ensureWarmed(holder, 8) } returns Unit every { store.startPump(holder) } returns Unit val result = SabrPlaybackSessionService(store).prepare("video", "user", prepared, audio, video, 0L) @@ -67,8 +66,8 @@ class SabrTransitioningLivePlaybackTest { assertTrue(holder.expectsLive()) assertTrue(result.startTimeMs > 0L) assertNull(holder.terminalFailure()) - assertEquals(11_050_200L, result.startTimeMs) - coVerify(exactly = 1) { store.ensureWarmed(holder, 8) } + assertEquals(11_068_200L, result.startTimeMs) + coVerify(exactly = 0) { store.ensureWarmed(any(), any()) } verify(exactly = 1) { state.setPlayerTimeMs(9_007_199_254_740_991L) } verify(exactly = 1) { store.startPump(holder) } } From 71d3892a782c48ed15b46f48af1ac78c2c785f98 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 21:14:47 +0000 Subject: [PATCH 078/137] fix: bypass cache for YouTube live HLS manifests --- .../typetype/server/services/HlsManifestService.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index e1f13d23..a4b62da9 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -49,7 +49,9 @@ class HlsManifestService( } private suspend fun cachedOrFetch(manifestUrl: String, signManifestLinks: Boolean): ExtractionResult { - val cache = manifestCache.takeUnless { isProviderManifestUrl(manifestUrl) } + val cache = manifestCache.takeUnless { + isProviderManifestUrl(manifestUrl) || isYoutubeLiveManifest(manifestUrl) + } val cacheKey = if (signManifestLinks) "signed:$manifestUrl" else manifestUrl cache?.get(cacheKey)?.let { return ExtractionResult.Success(it) } val pending = CompletableDeferred>() @@ -158,4 +160,11 @@ class HlsManifestService( private fun isProviderManifestUrl(url: String): Boolean = isNicoNicoManifest(url) || isBilibiliManifest(url) + + private fun isYoutubeLiveManifest(url: String): Boolean = runCatching { + val uri = URI(url) + val host = uri.host?.lowercase().orEmpty() + val isYoutubeHost = host == "googlevideo.com" || host.endsWith(".googlevideo.com") + isYoutubeHost && uri.path.orEmpty().split('/').any { it == "yt_live_broadcast" } + }.getOrDefault(false) } From 919fc7e5186ea11dbd8e9259eae3e2cf66fed1cb Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 21:44:34 +0000 Subject: [PATCH 079/137] refactor: split stream route helpers --- openapi/components/streams.yaml | 4 +- openapi/paths/streams.yaml | 8 +- .../server/routes/SabrStreamContractFilter.kt | 12 +++ .../server/routes/StreamRouteHelpers.kt | 82 +++++++++++++++++++ .../typetype/server/routes/StreamRoutes.kt | 74 +---------------- .../dev/typetype/server/StreamRoutesTest.kt | 39 +++++++++ 6 files changed, 142 insertions(+), 77 deletions(-) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt diff --git a/openapi/components/streams.yaml b/openapi/components/streams.yaml index 4ef4c5e2..1740e02b 100644 --- a/openapi/components/streams.yaml +++ b/openapi/components/streams.yaml @@ -59,7 +59,7 @@ SubtitleItem: isAutoGenerated: { type: boolean } StreamResponse: type: object - required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, isLive, isPostLive, isLiveContent, hasLiveManifest, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] + required: [id, title, uploaderName, uploaderUrl, thumbnailUrl, description, duration, viewCount, uploadDate, uploaded, streamType, isLive, isPostLive, isLiveContent, hasLiveManifest, hlsUrl, dashMpdUrl, videoStreams, audioStreams, videoOnlyStreams, subtitles, relatedStreams] properties: id: { type: string } title: { type: string } @@ -80,6 +80,8 @@ StreamResponse: isPostLive: { type: boolean } isLiveContent: { type: boolean } hasLiveManifest: { type: boolean } + hlsUrl: { type: string, description: HLS manifest URL, signed for server proxying when needed } + dashMpdUrl: { type: string, description: DASH manifest URL } isShortFormContent: { type: boolean } originalAudioTrackId: { type: string, nullable: true } preferredDefaultAudioTrackId: { type: string, nullable: true } diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index a0bab142..5a91f50f 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -1,7 +1,7 @@ YoutubeSabrStreams: get: tags: [extraction] - summary: Extract YouTube SABR stream metadata + summary: Resolve YouTube playback metadata with HLS for live streams and SABR for VOD parameters: - name: url in: query @@ -9,7 +9,7 @@ YoutubeSabrStreams: schema: { type: string } responses: '200': - description: Stream metadata containing only SABR sources. + description: Live streams contain only an HLS manifest; VOD responses contain only SABR sources. headers: X-Request-ID: $ref: ../components/common.yaml#/RequestIdHeader @@ -32,7 +32,7 @@ YoutubeSabrStreams: YoutubeSabrBootstrap: get: tags: [extraction] - summary: Resolve playable SABR audio and video formats before full metadata + summary: Resolve early playback metadata with HLS for live streams and SABR formats for VOD parameters: - name: url in: query @@ -40,7 +40,7 @@ YoutubeSabrBootstrap: schema: { type: string } responses: '200': - description: Minimal stream metadata containing only SABR sources. + description: Live streams contain only an HLS manifest; VOD responses contain only SABR sources. headers: X-Request-ID: $ref: ../components/common.yaml#/RequestIdHeader diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt index 9871b7fb..19135777 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrStreamContractFilter.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.AudioStreamItem import dev.typetype.server.models.StreamResponse import dev.typetype.server.models.VideoStreamItem import dev.typetype.server.services.SabrSessionStore +import dev.typetype.server.services.withSabrManifestUrls import dev.typetype.server.sabr.YoutubeSabrFormat import dev.typetype.server.sabr.YoutubeSabrInfo import java.net.URLEncoder @@ -60,6 +61,14 @@ internal fun StreamResponse.withoutSabrStreams(): StreamResponse = copy( audioStreams = audioStreams.filterNot { it.deliveryMethod == SABR_DELIVERY_METHOD }, ) +internal fun StreamResponse.onlyLiveHls(): StreamResponse = copy( + dashMpdUrl = "", + videoStreams = emptyList(), + videoOnlyStreams = emptyList(), + audioStreams = emptyList(), + hasLiveManifest = hlsUrl.isNotBlank(), +) + internal fun StreamResponse.onlySabrStreams(): StreamResponse = copy( hlsUrl = "", dashMpdUrl = "", @@ -151,3 +160,6 @@ private fun String.codec(): String? { } private const val SABR_DELIVERY_METHOD = "sabr" + +internal fun StreamResponse.forSabrPlayback(): StreamResponse = + if (isLive) onlyLiveHls() else withSabrManifestUrls().onlySabrStreams() diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt new file mode 100644 index 00000000..58c1352c --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt @@ -0,0 +1,82 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import dev.typetype.server.services.PublicHlsManifestTokenService +import dev.typetype.server.services.ProviderMediaType +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE +import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR +import dev.typetype.server.services.requiresYoutubeSession + +private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" +private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" +private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" + +internal fun streamCacheControl( + deliveryMode: StreamDeliveryMode, + isLive: Boolean, + userId: String?, +): String = when { + deliveryMode.isSabr() && isLive -> PROVIDER_STREAMS_CACHE_CONTROL + userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.NicoNico || + deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL + else -> STREAMS_CACHE_CONTROL +} + +internal fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { + StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO + StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI + StreamDeliveryMode.YoutubeSabr -> null +} + +internal data class StreamResolution( + val result: ExtractionResult, + val authenticated: Boolean = false, +) + +internal suspend fun resolveStreamInfo( + url: String, + deliveryMode: StreamDeliveryMode, + userId: String?, + publicResult: ExtractionResult, + dependencies: StreamRouteDependencies, +): StreamResolution { + if (deliveryMode == StreamDeliveryMode.BiliBili) { + val bilibiliInfo = dependencies.bilibiliSessionStreamInfo + if (bilibiliInfo != null && userId != null) { + val bilibiliResult = bilibiliInfo(userId, url) + if (bilibiliResult != null) return StreamResolution(bilibiliResult, authenticated = true) + } + return StreamResolution(publicResult) + } + val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo + if (!deliveryMode.isSabr() || authenticatedInfo == null) { + return StreamResolution(publicResult) + } + val authenticatedResult = userId?.let { authenticatedInfo(it, url) } + if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) + return if (publicResult.requiresYoutubeSession()) { + StreamResolution( + ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), + ) + } else { + StreamResolution(publicResult) + } +} + +internal fun StreamResponse.hasPlayableSource(): Boolean = + videoStreams.isNotEmpty() || + videoOnlyStreams.isNotEmpty() || + audioStreams.isNotEmpty() || + hlsUrl.isNotBlank() || + dashMpdUrl.isNotBlank() + +internal fun StreamResponse.withSignedPublicHlsUrl( + shouldSign: Boolean, + tokenService: PublicHlsManifestTokenService?, +): StreamResponse { + if (!shouldSign || tokenService == null || hlsUrl.isBlank()) return this + if (hlsUrl.startsWith("/streams/hls-manifest?token=")) return this + return copy(hlsUrl = tokenService.createPath(hlsUrl)) +} diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index 9fb17cb0..b2563c07 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -10,14 +10,9 @@ import dev.typetype.server.services.BlockedContentProfile import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.ProviderMediaHandleService -import dev.typetype.server.services.ProviderMediaType import dev.typetype.server.services.StreamService -import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_CODE -import dev.typetype.server.services.YOUTUBE_SESSION_REQUIRED_ERROR import dev.typetype.server.services.filterAllowed import dev.typetype.server.services.filterBlocked -import dev.typetype.server.services.requiresYoutubeSession -import dev.typetype.server.services.withSabrManifestUrls import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond @@ -25,9 +20,6 @@ import io.ktor.server.routing.Route import io.ktor.server.routing.get import kotlinx.coroutines.CancellationException -private const val STREAMS_CACHE_CONTROL = "public, max-age=21600, stale-while-revalidate=3600" -private const val AUTHENTICATED_STREAMS_CACHE_CONTROL = "no-store" -private const val PROVIDER_STREAMS_CACHE_CONTROL = "no-store" fun Route.streamRoutes( streamService: StreamService, @@ -110,7 +102,7 @@ private fun Route.streamRoute( ) } val selected = if (deliveryMode.isSabr()) { - result.data.withSabrManifestUrls().onlySabrStreams() + result.data.forSabrPlayback() } else { result.data.withoutSabrStreams() } @@ -156,12 +148,7 @@ private fun Route.streamRoute( } call.response.headers.append( HttpHeaders.CacheControl, - when { - access.userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL - deliveryMode == StreamDeliveryMode.NicoNico || - deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL - else -> STREAMS_CACHE_CONTROL - }, + streamCacheControl(deliveryMode, selected.isLive, access.userId), ) call.respond(publicData) } @@ -172,60 +159,3 @@ private fun Route.streamRoute( } } } - -private fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaType? = when (deliveryMode) { - StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO - StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI - StreamDeliveryMode.YoutubeSabr -> null -} - -private data class StreamResolution( - val result: ExtractionResult, - val authenticated: Boolean = false, -) - -private suspend fun resolveStreamInfo( - url: String, - deliveryMode: StreamDeliveryMode, - userId: String?, - publicResult: ExtractionResult, - dependencies: StreamRouteDependencies, -): StreamResolution { - if (deliveryMode == StreamDeliveryMode.BiliBili) { - val bilibiliInfo = dependencies.bilibiliSessionStreamInfo - if (bilibiliInfo != null && userId != null) { - val bilibiliResult = bilibiliInfo(userId, url) - if (bilibiliResult != null) return StreamResolution(bilibiliResult, authenticated = true) - } - return StreamResolution(publicResult) - } - val authenticatedInfo = dependencies.youtubeSessionSabrStreamInfo - if (!deliveryMode.isSabr() || authenticatedInfo == null) { - return StreamResolution(publicResult) - } - val authenticatedResult = userId?.let { authenticatedInfo(it, url) } - if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) - return if (publicResult.requiresYoutubeSession()) { - StreamResolution( - ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), - ) - } else { - StreamResolution(publicResult) - } -} - -private fun StreamResponse.hasPlayableSource(): Boolean = - videoStreams.isNotEmpty() || - videoOnlyStreams.isNotEmpty() || - audioStreams.isNotEmpty() || - hlsUrl.isNotBlank() || - dashMpdUrl.isNotBlank() - -private fun StreamResponse.withSignedPublicHlsUrl( - shouldSign: Boolean, - tokenService: PublicHlsManifestTokenService?, -): StreamResponse { - if (!shouldSign || tokenService == null || hlsUrl.isBlank()) return this - if (hlsUrl.startsWith("/streams/hls-manifest?token=")) return this - return copy(hlsUrl = tokenService.createPath(hlsUrl)) -} diff --git a/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt index d9d6cac6..4c231589 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -15,6 +15,7 @@ import io.ktor.server.testing.testApplication import io.mockk.coEvery import io.mockk.mockk import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -45,6 +46,44 @@ class StreamRoutesTest { assertEquals("public, max-age=21600, stale-while-revalidate=3600", response.headers[HttpHeaders.CacheControl]) } + @Test + fun `GET live YouTube streams exposes HLS only`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( + sabrResponse().copy(isLive = true, hlsUrl = "/streams/hls-manifest?token=live"), + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=live") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?token=live\"")) + assertTrue(body.contains("\"dashMpdUrl\":\"\"")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) + } + + @Test + fun `GET live YouTube streams fails when HLS is unavailable instead of using SABR`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Success(sabrResponse().copy(isLive = true, hlsUrl = "")) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=live") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) + } + @Test fun `GET streams returns 422 on Failure`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns From a1821afe8e9a3146c0c86805709f81e6c626164b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 21:44:44 +0000 Subject: [PATCH 080/137] fix: resolve live YouTube playback with HLS --- .../services/SabrBootstrapStreamService.kt | 2 + .../services/SabrFallbackStreamService.kt | 8 ++- .../SabrBootstrapStreamLiveHlsTest.kt | 36 +++++++++++++ .../services/SabrCachedSegmentLocatorTest.kt | 4 +- .../server/ExtractionServiceRegistry.kt | 16 ++++-- .../services/YoutubeLiveHlsStreamService.kt | 25 +++++++++ .../YoutubeSessionSabrStreamService.kt | 1 + .../YoutubeLiveHlsStreamServiceTest.kt | 53 +++++++++++++++++++ .../YoutubeSessionSabrStreamServiceTest.kt | 33 ++++++++++++ .../services/SabrInitializationPolicyTest.kt | 6 +-- 10 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt index a50ec0c5..d7b3d746 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -6,12 +6,14 @@ import dev.typetype.server.models.StreamResponse class SabrBootstrapStreamService( private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, + private val liveHlsStreamService: StreamService, ) : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult { val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") val metadata = tokenSessionClient.fetchPlaybackSession(videoId) ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") + if (metadata.isLive) return liveHlsStreamService.getStreamInfo(url) val prepared = metadata.preparedSabrInfo() ?: sessionStore.fetchInfo(videoId, cachedFirst = true) ?: return ExtractionResult.Failure("SABR playback formats unavailable") diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt index 342b2af7..bf01f9e3 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt @@ -16,14 +16,18 @@ class SabrFallbackStreamService( val prepared = videoId?.let { async { sessionStore.fetchInfo(it, cachedFirst = true) } } val result = delegate.getStreamInfo(url) val response = (result as? ExtractionResult.Success)?.data + if (response?.isLive == true) { + prepared?.cancel() + return@coroutineScope result + } if (response == null) { if (result !is ExtractionResult.Failure || videoId == null) return@coroutineScope result prepared?.await() val session = tokenSessionClient.fetchPlaybackSession(videoId) ?: return@coroutineScope result return@coroutineScope ExtractionResult.Success(session.toFallbackStreamResponse(videoId)) } - val playable = prepared?.await() - if (response.hasSabrStreams() || videoId == null || playable == null) return@coroutineScope result + if (response.hasSabrStreams() || videoId == null) return@coroutineScope result + val playable = prepared?.await() ?: return@coroutineScope result ExtractionResult.Success(response.withSabrFallback(videoId, playable.info)) } } diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt new file mode 100644 index 00000000..cd091f31 --- /dev/null +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrBootstrapStreamLiveHlsTest.kt @@ -0,0 +1,36 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class SabrBootstrapStreamLiveHlsTest { + @Test + fun usesLiveHlsExtractionWithoutPreparingSabr() = runTest { + val sessionStore = mockk() + val tokenClient = mockk() + val liveHlsService = mockk() + val liveMetadata = mockk() + every { liveMetadata.isLive } returns true + val expected = ExtractionResult.Success(mockk()) + coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns liveMetadata + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = SabrBootstrapStreamService(sessionStore, tokenClient, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + coVerify(exactly = 0) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 0) { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + } +} diff --git a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt index feea11e9..a2a04d39 100644 --- a/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt +++ b/server-playback/src/test/kotlin/dev/typetype/server/services/SabrCachedSegmentLocatorTest.kt @@ -20,7 +20,7 @@ class SabrBootstrapStreamServiceTest { val session = tokenSession(prepared.info, tokenBundle()) coEvery { sessionStore.rememberPreparedInfo(VIDEO_ID, any()) } returns Unit coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns session - val service = SabrBootstrapStreamService(sessionStore, tokenClient) + val service = SabrBootstrapStreamService(sessionStore, tokenClient, mockk()) val result = service.getStreamInfo(YOUTUBE_URL) @@ -40,7 +40,7 @@ class SabrBootstrapStreamServiceTest { val prepared = preparedInfo() coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns null coEvery { tokenClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession(prepared.info) - val service = SabrBootstrapStreamService(sessionStore, tokenClient) + val service = SabrBootstrapStreamService(sessionStore, tokenClient, mockk()) val result = service.getStreamInfo(YOUTUBE_URL) diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 135d47a1..7898d14f 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -44,6 +44,7 @@ import dev.typetype.server.services.StreamYouTubeSubtitleResolver import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient import dev.typetype.server.services.YoutubePlayerClientStreamService +import dev.typetype.server.services.YoutubeLiveHlsStreamService import dev.typetype.server.services.YoutubeScopedChannelService import dev.typetype.server.services.YoutubeScopedCommentService import dev.typetype.server.services.YoutubeScopedPublicPlaylistService @@ -104,9 +105,13 @@ internal class ExtractionServiceRegistry( directPipePipeStreamService, YoutubePlayerClient.VISIONOS, ) - private val authenticatedStreamService = YoutubePlayerClientStreamService( + private val liveHlsStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, - YoutubePlayerClient.MWEB, + YoutubePlayerClient.TV_DOWNGRADED, + ) + private val authenticatedStreamService = YoutubeLiveHlsStreamService( + YoutubePlayerClientStreamService(directPipePipeStreamService, YoutubePlayerClient.MWEB), + liveHlsStreamService, ) private val sabrPublicStreamService = YoutubePlayerClientStreamService( sabrPipePipeStreamService, @@ -127,7 +132,10 @@ internal class ExtractionServiceRegistry( } val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( - SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), + YoutubeLiveHlsStreamService( + SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), + liveHlsStreamService, + ), ), cache, "stream-youtube-sabr:v1", @@ -142,7 +150,7 @@ internal class ExtractionServiceRegistry( YouTubeSubtitleCache(cache), ) val youtubeSabrBootstrapStreamService = YoutubeScopedStreamService( - SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient), + SabrBootstrapStreamService(sabrSessionStore, tokenYoutubeSessionClient, liveHlsStreamService), ) val nicoNicoStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-niconico:v1") val bilibiliStreamService = CachedStreamService(directPipePipeStreamService, cache, "stream-bilibili:v1") diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt new file mode 100644 index 00000000..acd9a1f7 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt @@ -0,0 +1,25 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class YoutubeLiveHlsStreamService( + private val metadataService: StreamService, + private val liveHlsService: StreamService, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return metadataService.getStreamInfo(url) + + val metadataResult = metadataService.getStreamInfo(url) + val metadata = (metadataResult as? ExtractionResult.Success)?.data ?: return metadataResult + if (!metadata.isLive || metadata.hlsUrl.isNotBlank()) return metadataResult + + val liveResult = liveHlsService.getStreamInfo(url) + val liveResponse = (liveResult as? ExtractionResult.Success)?.data + return if (liveResponse?.isLive == true && liveResponse.hlsUrl.isNotBlank()) { + liveResult + } else { + metadataResult + } + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index 7de1ffde..6ec55497 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -18,6 +18,7 @@ class YoutubeSessionSabrStreamService( withTimeout(timeoutMs) { val metadata = metadataService.getStreamInfo(userId, url) ?: return@withTimeout null if (metadata !is ExtractionResult.Success) return@withTimeout metadata + if (metadata.data.isLive) return@withTimeout metadata val videoId = youtubeVideoId(url) ?: return@withTimeout ExtractionResult.BadRequest("Invalid YouTube URL") when (val info = infoService.fetch(userId, videoId)) { is AuthenticatedSabrInfoResult.Ready -> diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..418cddae --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeLiveHlsStreamServiceTest { + @Test + fun retriesActiveLiveWithLiveHlsClientWhenMwebHasNoManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "") + val liveHls = streamResponse(isLive = true, manifestUrl = HLS_URL) + val expected = ExtractionResult.Success(liveHls) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(metadata) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun doesNotRetryVod() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val vod = streamResponse(isLive = false, manifestUrl = "") + val expected = ExtractionResult.Success(vod) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 0) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + private fun streamResponse(isLive: Boolean, manifestUrl: String): StreamResponse { + val response = mockk() + every { response.isLive } returns isLive + every { response.hlsUrl } returns manifestUrl + return response + } + + private companion object { + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=GlzleRbo5E0" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt new file mode 100644 index 00000000..cae68fe4 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt @@ -0,0 +1,33 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeSessionSabrStreamServiceTest { + @Test + fun returnsLiveMetadataWithoutPreparingSabr() = runTest { + val metadataService = mockk() + val infoService = mockk() + val live = mockk() + every { live.isLive } returns true + val expected = ExtractionResult.Success(live) + coEvery { metadataService.getStreamInfo(USER_ID, YOUTUBE_URL) } returns expected + val service = YoutubeSessionSabrStreamService(metadataService, infoService) + + assertSame(expected, service.getStreamInfo(USER_ID, YOUTUBE_URL)) + coVerify(exactly = 0) { infoService.fetch(USER_ID, VIDEO_ID) } + } + + private companion object { + const val USER_ID = "user-id" + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + } +} diff --git a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt index c5c0c8f0..20fd00e9 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt @@ -57,7 +57,7 @@ class SabrFallbackStreamServiceTest { } @Test - fun `enriches a live hls extraction with prepared sabr formats`() = runTest { + fun `keeps live hls extraction on hls without sabr fallback`() = runTest { val delegate = mockk() val sessionStore = mockk() val tokenSessionClient = mockk() @@ -74,8 +74,8 @@ class SabrFallbackStreamServiceTest { val enriched = (result as ExtractionResult.Success).data assertEquals(LIVE_HLS_URL, enriched.hlsUrl) - assertEquals(listOf(137), enriched.videoOnlyStreams.map { it.itag }) - assertEquals(listOf(140), enriched.audioStreams.map { it.itag }) + assertTrue(enriched.videoOnlyStreams.isEmpty()) + assertTrue(enriched.audioStreams.isEmpty()) } @Test From 433d509e68d55a06cf2fa826f106d18b579d9bee Mon Sep 17 00:00:00 2001 From: Priveetee Date: Wed, 23 Sep 2026 21:44:53 +0000 Subject: [PATCH 081/137] perf: retain live HLS manifest tokens longer --- .../server/services/OkHttpProxyService.kt | 3 +- .../services/PublicHlsManifestTokenService.kt | 2 +- .../services/SignedHlsManifestTokenService.kt | 2 +- .../typetype/server/OkHttpProxyServiceTest.kt | 75 +++++++++++++++++++ .../SignedHlsManifestTokenServiceTest.kt | 8 +- .../server/HlsManifestServiceCacheTest.kt | 27 +++++++ .../ManifestHlsPublicTokenRoutesTest.kt | 12 ++- .../server/StreamRoutesDeliveryModeTest.kt | 10 ++- .../server/StreamRoutesSignedHlsTest.kt | 10 ++- 9 files changed, 136 insertions(+), 13 deletions(-) create mode 100644 server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 43b256c6..0eea6e4c 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -9,6 +9,7 @@ import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request import java.io.ByteArrayInputStream +import java.net.URI import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.util.concurrent.TimeUnit @@ -156,7 +157,7 @@ class OkHttpProxyService( } private fun isHls(contentType: String, url: String): Boolean = - contentType.contains("mpegurl", ignoreCase = true) || url.contains(".m3u8", ignoreCase = true) + contentType.contains("mpegurl", ignoreCase = true) || runCatching { URI(url).path?.endsWith(".m3u8", ignoreCase = true) == true }.getOrDefault(false) companion object { private const val BILIBILI_REFERER = "https://www.bilibili.com" diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt index b9b5fb9b..9563e378 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PublicHlsManifestTokenService.kt @@ -50,7 +50,7 @@ class PublicHlsManifestTokenService( private fun decode(value: String): String? = runCatching { String(decoder.decode(value), Charsets.UTF_8) }.getOrNull() companion object { - const val TTL_SECONDS = 900L + const val TTL_SECONDS = 86_400L private const val VERSION = "ph1" private const val PART_COUNT = 4 private const val MAX_TOKEN_LENGTH = 4096 diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt index ba3f8fc7..3c39f9d4 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SignedHlsManifestTokenService.kt @@ -67,7 +67,7 @@ class SignedHlsManifestTokenService( runCatching { String(decoder.decode(value), Charsets.UTF_8) }.getOrNull() companion object { - const val TTL_SECONDS = 900L + const val TTL_SECONDS = 86_400L private const val VERSION = "v1" private const val PART_COUNT = 6 private const val MAX_TOKEN_LENGTH = 4096 diff --git a/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt new file mode 100644 index 00000000..4c1a934c --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceTest.kt @@ -0,0 +1,75 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.OkHttpProxyService +import kotlinx.coroutines.test.runTest +import okhttp3.Dns +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class OkHttpProxyServiceTest { + @Test + fun `live HLS segment URL containing m3u8 path preserves binary bytes`() = runTest { + val mediaBytes = byteArrayOf( + 0x47, 0x40, 0x00, 0x30, 0xff.toByte(), 0x00, 0xff.toByte(), 0x10, + 0x80.toByte(), 0x7f, 0x00, 0x01, + ) + val service = OkHttpProxyService(clientFor(mediaBytes)) + val result = service.pipe( + "https://rr8---sn-n4g-jqbe6.googlevideo.com/videoplayback/playlist/index.m3u8/sq/42/file/seg.ts", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + val response = (result as ExtractionResult.Success).data + try { + assertEquals(200, response.status) + assertArrayEquals(mediaBytes, response.stream.readBytes()) + } finally { + response.close() + } + } + + @Test + fun `m3u8 manifest path is still rewritten when upstream MIME is generic`() = runTest { + val manifest = "#EXTM3U\n#EXTINF:5.0,\nhttps://rr8---sn-n4g-jqbe6.googlevideo.com/segment.ts\n" + val service = OkHttpProxyService(clientFor(manifest.toByteArray())) + val result = service.pipe( + "https://manifest.googlevideo.com/api/manifest/hls_variant/playlist/index.m3u8", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + val response = (result as ExtractionResult.Success).data + try { + assertTrue(response.stream.readBytes().decodeToString().contains("/proxy?url=")) + } finally { + response.close() + } + } + + private fun clientFor(body: ByteArray): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(Interceptor { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .header("Content-Type", "application/octet-stream") + .body(body.toResponseBody("application/octet-stream".toMediaType())) + .build() + }) + .build() +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt index 3eec1114..60210b1a 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/SignedHlsManifestTokenServiceTest.kt @@ -17,7 +17,7 @@ class SignedHlsManifestTokenServiceTest { assertEquals("user-1", payload.userId) assertEquals("https://youtube.com/watch?v=test", payload.videoUrl) assertEquals("fingerprint", payload.fingerprint) - assertEquals(901_000L, payload.expiresAt) + assertEquals(1_000L + SignedHlsManifestTokenService.TTL_SECONDS * 1_000L, payload.expiresAt) } @Test @@ -28,11 +28,13 @@ class SignedHlsManifestTokenServiceTest { } @Test - fun `token expires`() { + fun `token remains valid for a long live session and then expires`() { var now = 1_000L val service = SignedHlsManifestTokenService("secret", nowMillis = { now }) val token = service.createToken("user-1", "https://youtube.com/watch?v=test", "fingerprint") - now = 901_000L + now = 1_801_000L + assertTrue(service.verify(token) is SignedHlsManifestTokenResult.Valid) + now = 1_000L + SignedHlsManifestTokenService.TTL_SECONDS * 1_000L assertEquals(SignedHlsManifestTokenResult.Expired, service.verify(token)) } } diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index 2a54a629..9ac771dd 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -14,6 +14,7 @@ import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.net.InetAddress @@ -41,6 +42,32 @@ class HlsManifestServiceCacheTest { assertEquals(1, calls) } + @Test + fun `youtube live manifests are fetched again instead of using the cached playlist`() = runTest { + var calls = 0 + val client = proxyTestClient(Interceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body( + "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:$calls\nsegment.ts" + .toResponseBody("application/vnd.apple.mpegurl".toMediaType()), + ) + .build() + }) + val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) + val url = "https://manifest.googlevideo.com/api/manifest/hls_variant/source/yt_live_broadcast/id/live-video/index.m3u8" + + val first = service.hlsManifest(url) + val second = service.hlsManifest(url) + + assertEquals(2, calls) + assertNotEquals(first, second) + } + @Test fun `provider manifests are fetched again instead of serving signed cache entries`() = runTest { var calls = 0 diff --git a/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt b/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt index bfb280ee..c15385b9 100644 --- a/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/ManifestHlsPublicTokenRoutesTest.kt @@ -29,7 +29,8 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class ManifestHlsPublicTokenRoutesTest { - private val tokenService = PublicHlsManifestTokenService("test-secret") + private var now = 1_000L + private val tokenService = PublicHlsManifestTokenService("test-secret", nowMillis = { now }) private val cache = InMemoryPublicHlsCache() @Test @@ -49,6 +50,15 @@ class ManifestHlsPublicTokenRoutesTest { assertEquals(2, Regex("hls-manifest\\?token=").findAll(body).count()) assertTrue(body.contains("../proxy?url=")) assertFalse(body.contains("hls-manifest?url=")) + + val childToken = Regex("hls-manifest\\?token=([A-Za-z0-9._-]+)") + .find(body)?.groupValues?.get(1) + assertTrue(childToken != null) + now += 30 * 60 * 1_000L + + val childResponse = client.get("/streams/hls-manifest?token=$childToken") + + assertEquals(HttpStatusCode.OK, childResponse.status) } @Test diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index 143ff781..16414d01 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -75,7 +75,7 @@ class StreamRoutesDeliveryModeTest { } @Test - fun `sabr endpoint removes direct live manifest`() = testApplication { + fun `sabr endpoint exposes only HLS for live streams`() = testApplication { val live = sabrResponse().copy( hlsUrl = "/streams/hls-manifest?url=live", isLive = true, @@ -88,8 +88,12 @@ class StreamRoutesDeliveryModeTest { val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("\"hlsUrl\":\"\"")) - assertFalse(response.bodyAsText().contains("hls-manifest")) + assertTrue(response.bodyAsText().contains("\"hlsUrl\":\"/streams/hls-manifest?url=live\"")) + assertTrue(response.bodyAsText().contains("\"videoStreams\":[]")) + assertTrue(response.bodyAsText().contains("\"videoOnlyStreams\":[]")) + assertTrue(response.bodyAsText().contains("\"audioStreams\":[]")) + assertFalse(response.bodyAsText().contains("\"deliveryMethod\":\"sabr\"")) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) } @Test diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt index d2fa50ba..b39af026 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesSignedHlsTest.kt @@ -63,7 +63,7 @@ class StreamRoutesSignedHlsTest { } @Test - fun `anonymous sabr removes live hls url`() = testApplication { + fun `anonymous sabr live response keeps only signed HLS`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns ExtractionResult.Success( publicHlsStream().copy( videoOnlyStreams = listOf( @@ -83,9 +83,13 @@ class StreamRoutesSignedHlsTest { val body = response.bodyAsText() assertEquals(HttpStatusCode.OK, response.status) - assertTrue(body.contains("\"hlsUrl\":\"\"")) - assertFalse(body.contains("hls-manifest")) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?token=")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertFalse(body.contains("\"deliveryMethod\":\"sabr\"")) assertFalse(body.contains(MANIFEST_URL)) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) } @Test From c48e31e36772030febac4621c7700bb75c88c5c6 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 00:24:47 +0000 Subject: [PATCH 082/137] fix: avoid caching live stream URLs --- .../server/services/StreamCacheTtlResolver.kt | 1 + .../server/StreamCacheTtlResolverTest.kt | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt index f8d757f4..111e8dca 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamCacheTtlResolver.kt @@ -12,6 +12,7 @@ private const val SIGNED_STREAM_DEADLINE_SAFETY_SECONDS = 300L private const val MIN_CACHEABLE_STREAM_TTL_SECONDS = 60L fun StreamResponse.streamCacheTtlSeconds(nowEpochSeconds: Long = System.currentTimeMillis() / 1000): Long { + if (isLive || isLiveContent) return 0L val deadline = signedMediaUrls().mapNotNull { it.bilibiliDeadline() }.minOrNull() ?: return stableMetadataTtlSeconds() val ttl = deadline - nowEpochSeconds - SIGNED_STREAM_DEADLINE_SAFETY_SECONDS diff --git a/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt index 72ebc0aa..e72445c5 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/StreamCacheTtlResolverTest.kt @@ -18,6 +18,16 @@ class StreamCacheTtlResolverTest { assertEquals(300L, response("https://example.com/video.mp4", dislikeCount = -1L).streamCacheTtlSeconds(nowEpochSeconds = 1_000L)) } + @Test + fun `live content urls are not cached across clients`() { + val response = response( + "https://example.com/video.mp4", + isLive = true, + hlsUrl = "/streams/hls-manifest?token=live", + ) + assertEquals(0L, response.streamCacheTtlSeconds(nowEpochSeconds = 1_000L)) + } + @Test fun `bilibili stream ttl follows signed deadline`() { val url = "https://upos-hz-mirrorakam.akamaized.net/video.m4s?deadline=10000&upsig=x" @@ -42,7 +52,12 @@ class StreamCacheTtlResolverTest { assertEquals(1_700L, response(url).streamCacheTtlSeconds(nowEpochSeconds = 8_000L)) } - private fun response(url: String, dislikeCount: Long = 0L): StreamResponse = StreamResponse( + private fun response( + url: String, + dislikeCount: Long = 0L, + isLive: Boolean = false, + hlsUrl: String = "", + ): StreamResponse = StreamResponse( id = "id", title = "title", uploaderName = "uploader", @@ -67,12 +82,14 @@ class StreamCacheTtlResolverTest { requiresMembership = false, startPosition = 0L, streamSegments = emptyList(), - hlsUrl = "", + hlsUrl = hlsUrl, dashMpdUrl = "", videoStreams = emptyList(), audioStreams = emptyList(), originalAudioTrackId = null, preferredDefaultAudioTrackId = null, + isLive = isLive, + isLiveContent = isLive, videoOnlyStreams = listOf(video(url)), subtitles = emptyList(), previewFrames = emptyList(), From 8dc76a4c1d4bf5216fc5f5369b530dbbee1099d1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 00:57:02 +0000 Subject: [PATCH 083/137] fix: use public live HLS for sessions --- .../server/services/YoutubeSessionSabrStreamService.kt | 2 +- .../server/services/YoutubeSessionSabrStreamServiceTest.kt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index 6ec55497..cbb67120 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -18,7 +18,7 @@ class YoutubeSessionSabrStreamService( withTimeout(timeoutMs) { val metadata = metadataService.getStreamInfo(userId, url) ?: return@withTimeout null if (metadata !is ExtractionResult.Success) return@withTimeout metadata - if (metadata.data.isLive) return@withTimeout metadata + if (metadata.data.isLive) return@withTimeout null val videoId = youtubeVideoId(url) ?: return@withTimeout ExtractionResult.BadRequest("Invalid YouTube URL") when (val info = infoService.fetch(userId, videoId)) { is AuthenticatedSabrInfoResult.Ready -> diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt index cae68fe4..87a7dfa4 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamServiceTest.kt @@ -7,12 +7,12 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test class YoutubeSessionSabrStreamServiceTest { @Test - fun returnsLiveMetadataWithoutPreparingSabr() = runTest { + fun `returns null for live metadata so the public HLS path is used`() = runTest { val metadataService = mockk() val infoService = mockk() val live = mockk() @@ -21,7 +21,7 @@ class YoutubeSessionSabrStreamServiceTest { coEvery { metadataService.getStreamInfo(USER_ID, YOUTUBE_URL) } returns expected val service = YoutubeSessionSabrStreamService(metadataService, infoService) - assertSame(expected, service.getStreamInfo(USER_ID, YOUTUBE_URL)) + assertEquals(null, service.getStreamInfo(USER_ID, YOUTUBE_URL)) coVerify(exactly = 0) { infoService.fetch(USER_ID, VIDEO_ID) } } From 8edb7a7fa3d50686bb07b38b1b5a07a8eda0b3da Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 01:12:11 +0000 Subject: [PATCH 084/137] fix: prefer public live HLS for sessions --- .../dev/typetype/server/routes/StreamRouteHelpers.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt index 58c1352c..1bf06cd7 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt @@ -55,7 +55,13 @@ internal suspend fun resolveStreamInfo( return StreamResolution(publicResult) } val authenticatedResult = userId?.let { authenticatedInfo(it, url) } - if (authenticatedResult != null) return StreamResolution(authenticatedResult, authenticated = true) + if (authenticatedResult != null) { + val authenticatedLive = (authenticatedResult as? ExtractionResult.Success)?.data?.isLive == true + val publicLive = (publicResult as? ExtractionResult.Success)?.data?.isLive == true + if (!authenticatedLive || !publicLive) { + return StreamResolution(authenticatedResult, authenticated = true) + } + } return if (publicResult.requiresYoutubeSession()) { StreamResolution( ExtractionResult.BadRequest(YOUTUBE_SESSION_REQUIRED_ERROR, YOUTUBE_SESSION_REQUIRED_CODE), From 92d5825306c4e1cfabf2a1bd751f273c39f96f5e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 04:48:54 +0000 Subject: [PATCH 085/137] fix: prefer dedicated YouTube live HLS --- .../services/YoutubeLiveHlsStreamService.kt | 2 +- .../YoutubeLiveHlsStreamServiceTest.kt | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt index acd9a1f7..56094bca 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt @@ -12,7 +12,7 @@ class YoutubeLiveHlsStreamService( val metadataResult = metadataService.getStreamInfo(url) val metadata = (metadataResult as? ExtractionResult.Success)?.data ?: return metadataResult - if (!metadata.isLive || metadata.hlsUrl.isNotBlank()) return metadataResult + if (!metadata.isLive) return metadataResult val liveResult = liveHlsService.getStreamInfo(url) val liveResponse = (liveResult as? ExtractionResult.Success)?.data diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt index 418cddae..4eacce5b 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt @@ -26,6 +26,36 @@ class YoutubeLiveHlsStreamServiceTest { coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } } + @Test + fun prefersLiveHlsClientWhenMwebAlreadyHasManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "https://mweb.example/live.m3u8") + val liveHls = streamResponse(isLive = true, manifestUrl = HLS_URL) + val expected = ExtractionResult.Success(liveHls) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(metadata) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + + @Test + fun fallsBackToMwebManifestWhenLiveHlsClientHasNoManifest() = runTest { + val metadataService = mockk() + val liveHlsService = mockk() + val metadata = streamResponse(isLive = true, manifestUrl = "https://mweb.example/live.m3u8") + val liveHls = streamResponse(isLive = true, manifestUrl = "") + val expected = ExtractionResult.Success(metadata) + coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns expected + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(liveHls) + val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) + + assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + } + @Test fun doesNotRetryVod() = runTest { val metadataService = mockk() From c3b164859a44125049b62cc1c26503ed0e1c2e21 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 07:16:20 +0000 Subject: [PATCH 086/137] perf: add direct HLS route for known YouTube lives --- openapi.yaml | 1 + openapi/paths/streams.yaml | 27 ++++++++++++ .../server/ApplicationStreamRoutes.kt | 1 + .../server/routes/StreamDeliveryMode.kt | 2 + .../server/routes/StreamRouteHelpers.kt | 2 + .../typetype/server/routes/StreamRoutes.kt | 8 +++- .../server/ExtractionServiceRegistry.kt | 4 ++ .../dev/typetype/server/ServiceRegistry.kt | 1 + .../YoutubeDirectLiveHlsStreamService.kt | 23 ++++++++++ .../YoutubeDirectLiveHlsStreamServiceTest.kt | 44 +++++++++++++++++++ .../server/StreamRoutesDeliveryModeTest.kt | 34 +++++++++++++- 11 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt diff --git a/openapi.yaml b/openapi.yaml index 3b2c81c4..dc141567 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -22,6 +22,7 @@ paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } /streams/youtube/sabr: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrStreams } + /streams/youtube/live: { $ref: ./openapi/paths/streams.yaml#/YoutubeLiveHls } /streams/youtube/sabr/bootstrap: { $ref: ./openapi/paths/streams.yaml#/YoutubeSabrBootstrap } /streams/niconico: { $ref: ./openapi/paths/streams.yaml#/NicoNicoStreams } /streams/bilibili: { $ref: ./openapi/paths/streams.yaml#/BiliBiliStreams } diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index 5a91f50f..b4aeb7a6 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -29,6 +29,33 @@ YoutubeSabrStreams: $ref: ../components/common.yaml#/JsonError '422': $ref: ../components/common.yaml#/JsonError +YoutubeLiveHls: + get: + tags: [extraction] + summary: Resolve a known YouTube live stream directly to HLS + parameters: + - name: url + in: query + required: true + schema: { type: string } + responses: + '200': + description: Active live streams contain only an HLS manifest and no SABR or DASH sources. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/streams.yaml#/StreamResponse + '400': + $ref: ../components/common.yaml#/JsonError + '401': + $ref: ../components/common.yaml#/JsonError + '403': + $ref: ../components/common.yaml#/JsonError + '422': + $ref: ../components/common.yaml#/JsonError YoutubeSabrBootstrap: get: tags: [extraction] diff --git a/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt index 34740596..e6fd90b9 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/ApplicationStreamRoutes.kt @@ -25,6 +25,7 @@ internal fun Route.installStreamRoutes( streamService = svc.youtubeSabrStreamService, nicoNicoStreamService = svc.nicoNicoStreamService, bilibiliStreamService = svc.bilibiliStreamService, + youtubeLiveHlsStreamService = svc.youtubeLiveHlsStreamService, sabrBootstrapStreamService = svc.youtubeSabrBootstrapStreamService, authService = authService, accessControlService = svc.accessControlService, diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt index 6ab72c9e..a8640f93 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt @@ -4,6 +4,7 @@ import java.net.URI internal enum class StreamDeliveryMode { YoutubeSabr, + YoutubeLiveHls, NicoNico, BiliBili; @@ -11,6 +12,7 @@ internal enum class StreamDeliveryMode { val host = runCatching { URI(url).host?.lowercase() }.getOrNull() ?: return false return when (this) { YoutubeSabr -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" + YoutubeLiveHls -> host == "youtu.be" || host.endsWith(".youtube.com") || host == "youtube.com" NicoNico -> host == "nico.ms" || host.endsWith(".nicovideo.jp") || host == "nicovideo.jp" BiliBili -> host == "b23.tv" || host.endsWith(".bilibili.com") || host == "bilibili.com" } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt index 1bf06cd7..998045fb 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRouteHelpers.kt @@ -18,6 +18,7 @@ internal fun streamCacheControl( userId: String?, ): String = when { deliveryMode.isSabr() && isLive -> PROVIDER_STREAMS_CACHE_CONTROL + deliveryMode == StreamDeliveryMode.YoutubeLiveHls -> PROVIDER_STREAMS_CACHE_CONTROL userId != null -> AUTHENTICATED_STREAMS_CACHE_CONTROL deliveryMode == StreamDeliveryMode.NicoNico || deliveryMode == StreamDeliveryMode.BiliBili -> PROVIDER_STREAMS_CACHE_CONTROL @@ -28,6 +29,7 @@ internal fun providerMediaType(deliveryMode: StreamDeliveryMode): ProviderMediaT StreamDeliveryMode.NicoNico -> ProviderMediaType.NICONICO StreamDeliveryMode.BiliBili -> ProviderMediaType.BILIBILI StreamDeliveryMode.YoutubeSabr -> null + StreamDeliveryMode.YoutubeLiveHls -> null } internal data class StreamResolution( diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index b2563c07..c5cd2adf 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -32,6 +32,7 @@ fun Route.streamRoutes( nicoNicoStreamService: StreamService = streamService, bilibiliStreamService: StreamService = streamService, sabrBootstrapStreamService: StreamService = streamService, + youtubeLiveHlsStreamService: StreamService = streamService, youtubeSessionSabrStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, bilibiliSessionStreamInfo: (suspend (String, String) -> ExtractionResult?)? = null, sabrStreamContractFilter: (suspend (String, StreamResponse) -> StreamResponse)? = null, @@ -48,6 +49,7 @@ fun Route.streamRoutes( bilibiliSessionStreamInfo = bilibiliSessionStreamInfo, ) streamRoute("/streams/youtube/sabr", StreamDeliveryMode.YoutubeSabr, streamService, dependencies) + streamRoute("/streams/youtube/live", StreamDeliveryMode.YoutubeLiveHls, youtubeLiveHlsStreamService, dependencies) streamRoute( "/streams/youtube/sabr/bootstrap", StreamDeliveryMode.YoutubeSabr, @@ -103,6 +105,8 @@ private fun Route.streamRoute( } val selected = if (deliveryMode.isSabr()) { result.data.forSabrPlayback() + } else if (deliveryMode == StreamDeliveryMode.YoutubeLiveHls) { + result.data.onlyLiveHls() } else { result.data.withoutSabrStreams() } @@ -110,7 +114,9 @@ private fun Route.streamRoute( .filterAllowed(accessProfile) .filterBlocked(blockedProfile) .withSignedPublicHlsUrl( - deliveryMode.isSabr() && selected.isLive || access.userId != null && !access.allowGuest, + deliveryMode.isSabr() && selected.isLive || + deliveryMode == StreamDeliveryMode.YoutubeLiveHls && selected.isLive || + access.userId != null && !access.allowGuest, dependencies.publicHlsManifestTokenService, ) val data = if ( diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 7898d14f..bb7f7733 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -45,6 +45,7 @@ import dev.typetype.server.services.TokenYouTubeSubtitleContentFetcher import dev.typetype.server.services.YoutubePlayerClient import dev.typetype.server.services.YoutubePlayerClientStreamService import dev.typetype.server.services.YoutubeLiveHlsStreamService +import dev.typetype.server.services.YoutubeDirectLiveHlsStreamService import dev.typetype.server.services.YoutubeScopedChannelService import dev.typetype.server.services.YoutubeScopedCommentService import dev.typetype.server.services.YoutubeScopedPublicPlaylistService @@ -140,6 +141,9 @@ internal class ExtractionServiceRegistry( cache, "stream-youtube-sabr:v1", ) + val youtubeLiveHlsStreamService = YoutubeScopedStreamService( + YoutubeDirectLiveHlsStreamService(liveHlsStreamService), + ) val youtubeSubtitleDeliveryService = YouTubeSubtitleDeliveryService( StreamYouTubeSubtitleResolver(youtubeSabrStreamService, youtubeSubtitleService::fetchSubtitleInventory), TokenYouTubeSubtitleContentFetcher( diff --git a/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index b70b918c..25ee3519 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -76,6 +76,7 @@ class ServiceRegistry( val youtubeSessionStreamService = extraction.youtubeSessionStreamService val youtubeSessionSabrStreamService = extraction.youtubeSessionSabrStreamService val youtubeSabrStreamService = extraction.youtubeSabrStreamService + val youtubeLiveHlsStreamService = extraction.youtubeLiveHlsStreamService val youtubeSabrBootstrapStreamService = extraction.youtubeSabrBootstrapStreamService val nicoNicoStreamService = extraction.nicoNicoStreamService val bilibiliStreamService = extraction.bilibiliStreamService diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt new file mode 100644 index 00000000..7feb566b --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt @@ -0,0 +1,23 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse + +class YoutubeDirectLiveHlsStreamService( + private val liveHlsService: StreamService, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") + + return when (val result = liveHlsService.getStreamInfo(url)) { + is ExtractionResult.Success -> + if (result.data.isLive && result.data.hlsUrl.isNotBlank()) { + result + } else { + ExtractionResult.Failure("No active YouTube live stream is available", "live_stream_unavailable") + } + is ExtractionResult.BadRequest -> result + is ExtractionResult.Failure -> result + } + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..9f2598ed --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt @@ -0,0 +1,44 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Test + +class YoutubeDirectLiveHlsStreamServiceTest { + @Test + fun returnsLiveWithHlsWithoutMetadataRequest() = runTest { + val liveHlsService = mockk() + val response = mockk() + every { response.isLive } returns true + every { response.hlsUrl } returns HLS_URL + val expected = ExtractionResult.Success(response) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected + + assertSame(expected, YoutubeDirectLiveHlsStreamService(liveHlsService).getStreamInfo(YOUTUBE_URL)) + } + + @Test + fun rejectsNonLiveResult() = runTest { + val liveHlsService = mockk() + val response = mockk() + every { response.isLive } returns false + every { response.hlsUrl } returns HLS_URL + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Success(response) + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService).getStreamInfo(YOUTUBE_URL) + + assertEquals("live_stream_unavailable", (result as ExtractionResult.Failure).code) + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index 16414d01..0ee59409 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -27,6 +27,7 @@ class StreamRoutesDeliveryModeTest { private val sabrService: StreamService = mockk() private val nicoNicoService: StreamService = mockk() private val bilibiliService: StreamService = mockk() + private val liveHlsService: StreamService = mockk() @Test fun `removed classic endpoints are not registered`() = testApplication { @@ -42,10 +43,10 @@ class StreamRoutesDeliveryModeTest { coEvery { sabrService.getStreamInfo(any()) } returns ExtractionResult.Success(mixedResponse()) var sabrFilterCalled = false application { - installRoutes { _, data -> + installRoutes(sabrFilter = { _, data -> sabrFilterCalled = true data - } + }) } val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") @@ -96,6 +97,33 @@ class StreamRoutesDeliveryModeTest { assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) } + @Test + fun knownLiveEndpointReturnsOnlyHls() = testApplication { + val live = mixedResponse().copy( + hlsUrl = "/streams/hls-manifest?url=live", + dashMpdUrl = "/manifest/live.mpd", + isLive = true, + isLiveContent = true, + hasLiveManifest = true, + ) + coEvery { liveHlsService.getStreamInfo(any()) } returns ExtractionResult.Success(live) + application { installRoutes(liveHls = liveHlsService) } + + val response = client.get("/streams/youtube/live?url=$VIDEO_URL") + val body = response.bodyAsText() + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(body.contains("\"hlsUrl\":\"/streams/hls-manifest?url=live\"")) + assertTrue(body.contains("\"dashMpdUrl\":\"\"")) + assertTrue(body.contains("\"videoStreams\":[]")) + assertTrue(body.contains("\"videoOnlyStreams\":[]")) + assertTrue(body.contains("\"audioStreams\":[]")) + assertTrue(body.contains("\"hasLiveManifest\":true")) + assertEquals("no-store", response.headers[HttpHeaders.CacheControl]) + coVerify(exactly = 1) { liveHlsService.getStreamInfo(VIDEO_URL) } + coVerify(exactly = 0) { sabrService.getStreamInfo(any()) } + } + @Test fun `provider endpoints use isolated services`() = testApplication { coEvery { nicoNicoService.getStreamInfo(any()) } returns ExtractionResult.Success(testStreamResponse()) @@ -152,6 +180,7 @@ class StreamRoutesDeliveryModeTest { private fun Application.installRoutes( providerMediaHandleService: ProviderMediaHandleService? = null, sabrFilter: suspend (String, StreamResponse) -> StreamResponse = { _, data -> data }, + liveHls: StreamService = liveHlsService, ) { install(ContentNegotiation) { json() } routing { @@ -159,6 +188,7 @@ class StreamRoutesDeliveryModeTest { streamService = sabrService, nicoNicoStreamService = nicoNicoService, bilibiliStreamService = bilibiliService, + youtubeLiveHlsStreamService = liveHls, providerMediaHandleService = providerMediaHandleService, sabrStreamContractFilter = sabrFilter, ) From 6dc140e8aaca9b2c8fcfdb204466fdababc8c69f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 07:42:23 +0000 Subject: [PATCH 087/137] fix: fall back when direct live extraction fails --- .../server/ExtractionServiceRegistry.kt | 12 ++++++++++-- .../YoutubeDirectLiveHlsStreamService.kt | 11 ++++++++++- .../YoutubeDirectLiveHlsStreamServiceTest.kt | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index bb7f7733..2640f1bb 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -125,6 +125,11 @@ internal class ExtractionServiceRegistry( ) private val hlsTokenService = youtubeSessionSecret?.let(::SignedHlsManifestTokenService) private val tokenYoutubeSessionClient = TypetypeTokenYoutubeSessionClient(subtitleServiceUrl, httpClient) + private val youtubeSabrFallbackStreamService = SabrFallbackStreamService( + sabrPublicStreamService, + sabrSessionStore, + tokenYoutubeSessionClient, + ) val youtubeSessionStreamService = hlsTokenService?.let { YoutubeSessionStreamService(authenticatedStreamService, youtubeSessionService, cache, it) } @@ -134,7 +139,7 @@ internal class ExtractionServiceRegistry( val youtubeSabrStreamService = CachedStreamService( YoutubeScopedStreamService( YoutubeLiveHlsStreamService( - SabrFallbackStreamService(sabrPublicStreamService, sabrSessionStore, tokenYoutubeSessionClient), + youtubeSabrFallbackStreamService, liveHlsStreamService, ), ), @@ -142,7 +147,10 @@ internal class ExtractionServiceRegistry( "stream-youtube-sabr:v1", ) val youtubeLiveHlsStreamService = YoutubeScopedStreamService( - YoutubeDirectLiveHlsStreamService(liveHlsStreamService), + YoutubeDirectLiveHlsStreamService( + liveHlsStreamService, + youtubeSabrFallbackStreamService, + ), ) val youtubeSubtitleDeliveryService = YouTubeSubtitleDeliveryService( StreamYouTubeSubtitleResolver(youtubeSabrStreamService, youtubeSubtitleService::fetchSubtitleInventory), diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt index 7feb566b..41a8b1ed 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.StreamResponse class YoutubeDirectLiveHlsStreamService( private val liveHlsService: StreamService, + private val fallbackService: StreamService? = null, ) : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult { if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") @@ -17,7 +18,15 @@ class YoutubeDirectLiveHlsStreamService( ExtractionResult.Failure("No active YouTube live stream is available", "live_stream_unavailable") } is ExtractionResult.BadRequest -> result - is ExtractionResult.Failure -> result + is ExtractionResult.Failure -> fallbackIfLiveHls(url, result) } } + private suspend fun fallbackIfLiveHls( + url: String, + failure: ExtractionResult.Failure, + ): ExtractionResult { + val fallback = fallbackService?.getStreamInfo(url) as? ExtractionResult.Success + return fallback?.takeIf { it.data.isLive && it.data.hlsUrl.isNotBlank() } ?: failure + } + } diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt index 9f2598ed..3b5bb7b2 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt @@ -36,6 +36,24 @@ class YoutubeDirectLiveHlsStreamServiceTest { assertEquals("live_stream_unavailable", (result as ExtractionResult.Failure).code) } + @Test + fun usesFallbackWhenDirectLiveExtractionFails() = runTest { + val liveHlsService = mockk() + val fallbackService = mockk() + val response = mockk() + val directFailure = ExtractionResult.Failure("provider blocked", "provider_access_blocked") + val expected = ExtractionResult.Success(response) + every { response.isLive } returns true + every { response.hlsUrl } returns HLS_URL + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns directFailure + coEvery { fallbackService.getStreamInfo(YOUTUBE_URL) } returns expected + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + } + private companion object { const val VIDEO_ID = "GlzleRbo5E0" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" From 92b33f8548adc10e740d63eafc5d803a4880560c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 08:45:16 +0000 Subject: [PATCH 088/137] perf: fetch live HLS without enumerating formats --- .../server/ExtractionServiceRegistry.kt | 3 +- .../services/PipePipeLiveHlsStreamService.kt | 108 ++++++++++++++++++ .../services/YoutubeLiveHlsStreamService.kt | 12 +- 3 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 2640f1bb..61411231 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -27,6 +27,7 @@ import dev.typetype.server.services.PipePipePodcastService import dev.typetype.server.services.PipePipePublicPlaylistService import dev.typetype.server.services.PipePipeSearchService import dev.typetype.server.services.PipePipeStreamService +import dev.typetype.server.services.PipePipeLiveHlsStreamService import dev.typetype.server.services.PipePipeSuggestionService import dev.typetype.server.services.PipePipeTrendingService import dev.typetype.server.services.ProviderMediaHandleService @@ -107,7 +108,7 @@ internal class ExtractionServiceRegistry( YoutubePlayerClient.VISIONOS, ) private val liveHlsStreamService = YoutubePlayerClientStreamService( - directPipePipeStreamService, + PipePipeLiveHlsStreamService(), YoutubePlayerClient.TV_DOWNGRADED, ) private val authenticatedStreamService = YoutubeLiveHlsStreamService( diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt new file mode 100644 index 00000000..285da922 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamService.kt @@ -0,0 +1,108 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.stream.StreamExtractor + +class PipePipeLiveHlsStreamService( + private val extractorForUrl: (String) -> StreamExtractor = ::createStreamExtractor, +) : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") + + return withContext(Dispatchers.IO) { + try { + val extractor = extractorForUrl(url) + val response = withTimeout(LIVE_EXTRACTION_TIMEOUT_MS) { + runPipePipeCall { + extractor.fetchPage() + val streamType = extractor.streamType + val type = streamType.toApiStreamType() + if (!type.isLiveStreamType()) return@runPipePipeCall null + + val hlsUrl = extractor.hlsUrl + if (hlsUrl.isBlank() || extractor.requiresMembership()) { + return@runPipePipeCall null + } + extractor.toLiveHlsResponse(type, hlsUrl) + } + } + response?.let { ExtractionResult.Success(it) } + ?: ExtractionResult.Failure( + "No active YouTube live stream is available", + "live_stream_unavailable", + ) + } catch (_: TimeoutCancellationException) { + ExtractionResult.Failure("YouTube live extraction timed out", "live_stream_unavailable") + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + StreamExtractionErrorMapper.map(error, sourceUrl = url) + } + } + } + + private fun StreamExtractor.toLiveHlsResponse(streamType: String, hlsUrl: String) = StreamResponse( + id = id, + title = name.orEmpty(), + uploaderName = optionalExtractorValue("") { uploaderName }, + uploaderUrl = optionalExtractorValue("") { uploaderUrl }, + uploaderAvatarUrl = optionalExtractorValue("") { uploaderAvatarUrl }, + thumbnailUrl = optionalExtractorValue("") { thumbnailUrl }, + description = optionalExtractorValue("") { description.content.orEmpty() }, + duration = optionalExtractorValue(0L) { length.coerceAtLeast(0L) }, + viewCount = optionalExtractorValue(-1L) { viewCount }, + likeCount = -1L, + dislikeCount = -1L, + uploadDate = "", + uploaded = -1L, + uploaderSubscriberCount = -1L, + uploaderVerified = false, + category = "", + license = "", + visibility = "public", + tags = emptyList(), + streamType = streamType, + isLive = true, + isPostLive = false, + isLiveContent = true, + hasLiveManifest = true, + isShortFormContent = optionalExtractorValue(false) { isShortFormContent }, + requiresMembership = false, + startPosition = optionalExtractorValue(0L) { timeStamp.coerceAtLeast(0L) }, + streamSegments = emptyList(), + hlsUrl = hlsUrl, + dashMpdUrl = "", + videoStreams = emptyList(), + audioStreams = emptyList(), + originalAudioTrackId = null, + preferredDefaultAudioTrackId = null, + videoOnlyStreams = emptyList(), + subtitles = emptyList(), + previewFrames = emptyList(), + sponsorBlockSegments = emptyList(), + relatedStreams = emptyList(), + ) + + private companion object { + const val LIVE_EXTRACTION_TIMEOUT_MS = 8_000L + + fun createStreamExtractor(url: String): StreamExtractor { + val service = NewPipe.getServiceByUrl(url) + return service.getStreamExtractor(service.streamLHFactory.fromUrl(url)) + } + + inline fun optionalExtractorValue(default: T, value: () -> T): T = + try { + value() + } catch (_: Exception) { + default + } + } +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt index 56094bca..bc43771f 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamService.kt @@ -10,16 +10,12 @@ class YoutubeLiveHlsStreamService( override suspend fun getStreamInfo(url: String): ExtractionResult { if (!isYoutubeUrl(url)) return metadataService.getStreamInfo(url) - val metadataResult = metadataService.getStreamInfo(url) - val metadata = (metadataResult as? ExtractionResult.Success)?.data ?: return metadataResult - if (!metadata.isLive) return metadataResult - val liveResult = liveHlsService.getStreamInfo(url) val liveResponse = (liveResult as? ExtractionResult.Success)?.data - return if (liveResponse?.isLive == true && liveResponse.hlsUrl.isNotBlank()) { - liveResult - } else { - metadataResult + if (liveResponse?.isLive == true && !liveResponse.requiresMembership && liveResponse.hlsUrl.isNotBlank()) { + return liveResult } + + return metadataService.getStreamInfo(url) } } From 8f866602b82587369c873c4b9a7a3a4afaa327cc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 08:45:16 +0000 Subject: [PATCH 089/137] test: cover fast PipePipe live HLS extraction --- .../PipePipeLiveHlsStreamServiceTest.kt | 73 +++++++++++++++++++ .../YoutubeLiveHlsStreamServiceTest.kt | 5 +- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt new file mode 100644 index 00000000..1796abfe --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/PipePipeLiveHlsStreamServiceTest.kt @@ -0,0 +1,73 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.ExtractionResult +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.runs +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.stream.StreamExtractor +import org.schabi.newpipe.extractor.stream.StreamType + +class PipePipeLiveHlsStreamServiceTest { + @Test + fun returnsLiveHlsWithoutEnumeratingFormats() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.LIVE_STREAM + every { extractor.hlsUrl } returns HLS_URL + every { extractor.requiresMembership() } returns false + every { extractor.id } returns VIDEO_ID + every { extractor.name } returns "Live title" + val service = PipePipeLiveHlsStreamService { extractor } + + val response = (service.getStreamInfo(YOUTUBE_URL) as ExtractionResult.Success).data + + assertEquals(HLS_URL, response.hlsUrl) + assertTrue(response.isLive) + assertTrue(response.hasLiveManifest) + assertTrue(response.videoStreams.isEmpty()) + assertTrue(response.audioStreams.isEmpty()) + verify(exactly = 1) { extractor.fetchPage() } + verify(exactly = 0) { extractor.audioStreams } + verify(exactly = 0) { extractor.videoStreams } + } + + @Test + fun rejectsVodBeforeReadingManifest() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.VIDEO_STREAM + val service = PipePipeLiveHlsStreamService { extractor } + + val result = service.getStreamInfo(YOUTUBE_URL) + + assertTrue(result is ExtractionResult.Failure) + assertEquals("live_stream_unavailable", (result as ExtractionResult.Failure).code) + verify(exactly = 0) { extractor.hlsUrl } + } + + @Test + fun doesNotExposeMembersOnlyManifest() = runTest { + val extractor = mockk(relaxed = true) + every { extractor.fetchPage() } just runs + every { extractor.streamType } returns StreamType.LIVE_STREAM + every { extractor.hlsUrl } returns HLS_URL + every { extractor.requiresMembership() } returns true + val service = PipePipeLiveHlsStreamService { extractor } + + val result = service.getStreamInfo(YOUTUBE_URL) + + assertTrue(result is ExtractionResult.Failure) + } + + private companion object { + const val VIDEO_ID = "GlzleRbo5E0" + const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" + const val HLS_URL = "https://manifest.googlevideo.com/live.m3u8" + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt index 4eacce5b..38cdbd50 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt @@ -57,16 +57,17 @@ class YoutubeLiveHlsStreamServiceTest { } @Test - fun doesNotRetryVod() = runTest { + fun fallsBackToMwebAfterFastProbeFindsVod() = runTest { val metadataService = mockk() val liveHlsService = mockk() val vod = streamResponse(isLive = false, manifestUrl = "") + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure(YOUTUBE_URL) val expected = ExtractionResult.Success(vod) coEvery { metadataService.getStreamInfo(YOUTUBE_URL) } returns expected val service = YoutubeLiveHlsStreamService(metadataService, liveHlsService) assertSame(expected, service.getStreamInfo(YOUTUBE_URL)) - coVerify(exactly = 0) { liveHlsService.getStreamInfo(YOUTUBE_URL) } + coVerify(exactly = 1) { liveHlsService.getStreamInfo(YOUTUBE_URL) } } private fun streamResponse(isLive: Boolean, manifestUrl: String): StreamResponse { From 73863fe85f08caa84f3e629a8d21c291e65b9efd Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 08:54:53 +0000 Subject: [PATCH 090/137] test: stub membership state in live service tests --- .../typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt index 38cdbd50..32f5125b 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeLiveHlsStreamServiceTest.kt @@ -74,6 +74,7 @@ class YoutubeLiveHlsStreamServiceTest { val response = mockk() every { response.isLive } returns isLive every { response.hlsUrl } returns manifestUrl + every { response.requiresMembership } returns false return response } From 685c974d63aac91b4ed399ee30b9748176ddfff1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 10:12:57 +0000 Subject: [PATCH 091/137] perf: log youtube live extraction stage timing --- .../YoutubeDirectLiveHlsStreamService.kt | 87 +++++++++++++++++-- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt index 41a8b1ed..b6a2c832 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt @@ -2,31 +2,100 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse +import org.slf4j.LoggerFactory class YoutubeDirectLiveHlsStreamService( private val liveHlsService: StreamService, private val fallbackService: StreamService? = null, ) : StreamService { + private val logger = LoggerFactory.getLogger(YoutubeDirectLiveHlsStreamService::class.java) + override suspend fun getStreamInfo(url: String): ExtractionResult { if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") - return when (val result = liveHlsService.getStreamInfo(url)) { - is ExtractionResult.Success -> - if (result.data.isLive && result.data.hlsUrl.isNotBlank()) { - result + val videoId = youtubeVideoId(url) ?: "unknown" + val directStartedAt = System.nanoTime() + val directResult = liveHlsService.getStreamInfo(url) + val directElapsedMs = elapsedMsSince(directStartedAt) + + return when (directResult) { + is ExtractionResult.Success -> { + if (directResult.data.isLive && directResult.data.hlsUrl.isNotBlank()) { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=direct_hls", + videoId, + directElapsedMs, + ) + directResult } else { - ExtractionResult.Failure("No active YouTube live stream is available", "live_stream_unavailable") + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=not_live", + videoId, + directElapsedMs, + ) + ExtractionResult.Failure( + "No active YouTube live stream is available", + "live_stream_unavailable", + ) } - is ExtractionResult.BadRequest -> result - is ExtractionResult.Failure -> fallbackIfLiveHls(url, result) + } + is ExtractionResult.BadRequest -> { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=bad_request code={}", + videoId, + directElapsedMs, + directResult.code, + ) + directResult + } + is ExtractionResult.Failure -> + fallbackIfLiveHls(url, videoId, directElapsedMs, directResult) } } + private suspend fun fallbackIfLiveHls( url: String, + videoId: String, + directElapsedMs: Long, failure: ExtractionResult.Failure, ): ExtractionResult { - val fallback = fallbackService?.getStreamInfo(url) as? ExtractionResult.Success - return fallback?.takeIf { it.data.isLive && it.data.hlsUrl.isNotBlank() } ?: failure + val fallback = fallbackService + if (fallback == null) { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=direct_failure directCode={}", + videoId, + directElapsedMs, + failure.code, + ) + return failure + } + + val fallbackStartedAt = System.nanoTime() + val fallbackResult = fallback.getStreamInfo(url) + val fallbackElapsedMs = elapsedMsSince(fallbackStartedAt) + val liveFallback = (fallbackResult as? ExtractionResult.Success) + ?.takeIf { it.data.isLive && it.data.hlsUrl.isNotBlank() } + val fallbackStatus = when (fallbackResult) { + is ExtractionResult.Success -> if (liveFallback != null) "live_hls" else "not_live" + is ExtractionResult.BadRequest -> "bad_request:${fallbackResult.code}" + is ExtractionResult.Failure -> "failure:${fallbackResult.code}" + } + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} directCode={} result={}", + videoId, + directElapsedMs, + fallbackElapsedMs, + directElapsedMs + fallbackElapsedMs, + failure.code, + fallbackStatus, + ) + return liveFallback ?: failure } + private fun elapsedMsSince(startedAt: Long): Long = + (System.nanoTime() - startedAt) / NANOS_PER_MILLI + + private companion object { + const val NANOS_PER_MILLI = 1_000_000L + } } From f6b8a7e0462d407b8f6b1116c04272235194d322 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 11:21:46 +0000 Subject: [PATCH 092/137] perf: hedge YouTube live HLS resolution --- .../YoutubeDirectLiveHlsStreamService.kt | 204 ++++++++++++++---- .../YoutubeDirectLiveHlsStreamServiceTest.kt | 57 ++++- 2 files changed, 207 insertions(+), 54 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt index b6a2c832..134623a0 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamService.kt @@ -2,6 +2,11 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.selects.select +import kotlinx.coroutines.withTimeoutOrNull import org.slf4j.LoggerFactory class YoutubeDirectLiveHlsStreamService( @@ -10,92 +15,201 @@ class YoutubeDirectLiveHlsStreamService( ) : StreamService { private val logger = LoggerFactory.getLogger(YoutubeDirectLiveHlsStreamService::class.java) - override suspend fun getStreamInfo(url: String): ExtractionResult { - if (!isYoutubeUrl(url)) return ExtractionResult.BadRequest("Invalid YouTube URL") + override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { + if (!isYoutubeUrl(url)) return@coroutineScope ExtractionResult.BadRequest("Invalid YouTube URL") val videoId = youtubeVideoId(url) ?: "unknown" + val totalStartedAt = System.nanoTime() val directStartedAt = System.nanoTime() - val directResult = liveHlsService.getStreamInfo(url) - val directElapsedMs = elapsedMsSince(directStartedAt) - - return when (directResult) { - is ExtractionResult.Success -> { - if (directResult.data.isLive && directResult.data.hlsUrl.isNotBlank()) { - logger.info( - "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=direct_hls", - videoId, - directElapsedMs, - ) - directResult - } else { - logger.info( - "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=not_live", - videoId, - directElapsedMs, - ) - ExtractionResult.Failure( - "No active YouTube live stream is available", - "live_stream_unavailable", - ) - } + val directAttempt = async { timedExtraction(liveHlsService, url, directStartedAt) } + val directBeforeHedge = withTimeoutOrNull(HEDGE_DELAY_MILLIS) { directAttempt.await() } + + if (directBeforeHedge != null) { + return@coroutineScope resolveDirect( + url = url, + videoId = videoId, + direct = directBeforeHedge, + totalStartedAt = totalStartedAt, + ) + } + + val fallback = fallbackService + if (fallback == null) { + return@coroutineScope resolveDirect( + url = url, + videoId = videoId, + direct = directAttempt.await(), + totalStartedAt = totalStartedAt, + ) + } + + val fallbackStartedAt = System.nanoTime() + val fallbackAttempt = async { timedExtraction(fallback, url, fallbackStartedAt) } + try { + val first = select { + directAttempt.onAwait { CompletedAttempt(AttemptSource.DIRECT, it) } + fallbackAttempt.onAwait { CompletedAttempt(AttemptSource.FALLBACK, it) } } - is ExtractionResult.BadRequest -> { + if (first.source == AttemptSource.FALLBACK && first.extraction.result.isLiveHls()) { + val totalElapsedMs = elapsedMsSince(totalStartedAt) + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} directCode=pending result=live_hls source=fallback", + videoId, + elapsedMsSince(directStartedAt), + first.extraction.elapsedMs, + totalElapsedMs, + ) + return@coroutineScope first.extraction.result + } + + val direct = if (first.source == AttemptSource.DIRECT) { + first.extraction + } else { + directAttempt.await() + } + resolveDirect( + url = url, + videoId = videoId, + direct = direct, + totalStartedAt = totalStartedAt, + fallbackAttempt = fallbackAttempt, + completedFallback = first.extraction.takeIf { first.source == AttemptSource.FALLBACK }, + fallbackStartedAt = fallbackStartedAt, + ) + } finally { + directAttempt.cancel() + fallbackAttempt.cancel() + } + } + + private suspend fun resolveDirect( + url: String, + videoId: String, + direct: TimedExtraction, + totalStartedAt: Long, + fallbackAttempt: Deferred? = null, + completedFallback: TimedExtraction? = null, + fallbackStartedAt: Long? = null, + ): ExtractionResult = when (val directResult = direct.result) { + is ExtractionResult.Success -> { + if (directResult.data.isLive && directResult.data.hlsUrl.isNotBlank()) { logger.info( - "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=bad_request code={}", + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=direct_hls", videoId, - directElapsedMs, - directResult.code, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), ) directResult + } else { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=not_live", + videoId, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), + ) + ExtractionResult.Failure( + "No active YouTube live stream is available", + "live_stream_unavailable", + ) } - is ExtractionResult.Failure -> - fallbackIfLiveHls(url, videoId, directElapsedMs, directResult) } + is ExtractionResult.BadRequest -> { + logger.info( + "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} result=bad_request code={}", + videoId, + direct.elapsedMs, + fallbackElapsedMs(completedFallback, fallbackStartedAt), + elapsedMsSince(totalStartedAt), + directResult.code, + ) + directResult + } + is ExtractionResult.Failure -> resolveFailure( + url = url, + videoId = videoId, + direct = direct, + failure = directResult, + totalStartedAt = totalStartedAt, + fallbackAttempt = fallbackAttempt, + completedFallback = completedFallback, + ) } - private suspend fun fallbackIfLiveHls( + private suspend fun resolveFailure( url: String, videoId: String, - directElapsedMs: Long, + direct: TimedExtraction, failure: ExtractionResult.Failure, + totalStartedAt: Long, + fallbackAttempt: Deferred?, + completedFallback: TimedExtraction?, ): ExtractionResult { val fallback = fallbackService if (fallback == null) { logger.info( - "youtube_live_resolution videoId={} directMs={} fallbackMs=0 result=direct_failure directCode={}", + "youtube_live_resolution videoId={} directMs={} fallbackMs=0 totalMs={} result=direct_failure directCode={}", videoId, - directElapsedMs, + direct.elapsedMs, + elapsedMsSince(totalStartedAt), failure.code, ) return failure } - val fallbackStartedAt = System.nanoTime() - val fallbackResult = fallback.getStreamInfo(url) - val fallbackElapsedMs = elapsedMsSince(fallbackStartedAt) - val liveFallback = (fallbackResult as? ExtractionResult.Success) + val extraction = completedFallback ?: fallbackAttempt?.await() ?: timedExtraction(fallback, url) + val liveFallback = (extraction.result as? ExtractionResult.Success) ?.takeIf { it.data.isLive && it.data.hlsUrl.isNotBlank() } - val fallbackStatus = when (fallbackResult) { + val fallbackStatus = when (val result = extraction.result) { is ExtractionResult.Success -> if (liveFallback != null) "live_hls" else "not_live" - is ExtractionResult.BadRequest -> "bad_request:${fallbackResult.code}" - is ExtractionResult.Failure -> "failure:${fallbackResult.code}" + is ExtractionResult.BadRequest -> "bad_request:${result.code}" + is ExtractionResult.Failure -> "failure:${result.code}" } logger.info( "youtube_live_resolution videoId={} directMs={} fallbackMs={} totalMs={} directCode={} result={}", videoId, - directElapsedMs, - fallbackElapsedMs, - directElapsedMs + fallbackElapsedMs, + direct.elapsedMs, + extraction.elapsedMs, + elapsedMsSince(totalStartedAt), failure.code, fallbackStatus, ) return liveFallback ?: failure } + private suspend fun timedExtraction( + service: StreamService, + url: String, + startedAt: Long = System.nanoTime(), + ): TimedExtraction { + val result = service.getStreamInfo(url) + return TimedExtraction(result, elapsedMsSince(startedAt)) + } + + private fun ExtractionResult.isLiveHls(): Boolean = + (this as? ExtractionResult.Success)?.data?.let { it.isLive && it.hlsUrl.isNotBlank() } == true + + private fun fallbackElapsedMs(completed: TimedExtraction?, startedAt: Long?): Long = + completed?.elapsedMs ?: startedAt?.let(::elapsedMsSince) ?: 0 + private fun elapsedMsSince(startedAt: Long): Long = (System.nanoTime() - startedAt) / NANOS_PER_MILLI + private data class TimedExtraction( + val result: ExtractionResult, + val elapsedMs: Long, + ) + + private data class CompletedAttempt( + val source: AttemptSource, + val extraction: TimedExtraction, + ) + + private enum class AttemptSource { DIRECT, FALLBACK } + private companion object { + const val HEDGE_DELAY_MILLIS = 100L const val NANOS_PER_MILLI = 1_000_000L } } diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt index 3b5bb7b2..28b34e83 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/services/YoutubeDirectLiveHlsStreamServiceTest.kt @@ -3,24 +3,31 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.mockk +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.delay import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertSame +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class YoutubeDirectLiveHlsStreamServiceTest { @Test - fun returnsLiveWithHlsWithoutMetadataRequest() = runTest { + fun returnsLiveWithHlsWithoutCallingFallback() = runTest { val liveHlsService = mockk() - val response = mockk() - every { response.isLive } returns true - every { response.hlsUrl } returns HLS_URL + val fallbackService = mockk() + val response = liveResponse() val expected = ExtractionResult.Success(response) coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns expected - assertSame(expected, YoutubeDirectLiveHlsStreamService(liveHlsService).getStreamInfo(YOUTUBE_URL)) + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + coVerify(exactly = 0) { fallbackService.getStreamInfo(YOUTUBE_URL) } } @Test @@ -40,11 +47,8 @@ class YoutubeDirectLiveHlsStreamServiceTest { fun usesFallbackWhenDirectLiveExtractionFails() = runTest { val liveHlsService = mockk() val fallbackService = mockk() - val response = mockk() val directFailure = ExtractionResult.Failure("provider blocked", "provider_access_blocked") - val expected = ExtractionResult.Success(response) - every { response.isLive } returns true - every { response.hlsUrl } returns HLS_URL + val expected = ExtractionResult.Success(liveResponse()) coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } returns directFailure coEvery { fallbackService.getStreamInfo(YOUTUBE_URL) } returns expected @@ -54,6 +58,41 @@ class YoutubeDirectLiveHlsStreamServiceTest { assertSame(expected, result) } + @Test + fun startsFallbackAfterHedgeDelayAndCancelsSlowDirectAttempt() = runTest { + val liveHlsService = mockk() + val fallbackService = mockk() + val directCancelled = CompletableDeferred() + val fallbackStartedAt = CompletableDeferred() + val scheduler = testScheduler + val expected = ExtractionResult.Success(liveResponse()) + coEvery { liveHlsService.getStreamInfo(YOUTUBE_URL) } coAnswers { + try { + delay(10_000) + ExtractionResult.Failure("late direct failure", "provider_access_blocked") + } finally { + directCancelled.complete(Unit) + } + } + coEvery { fallbackService.getStreamInfo(YOUTUBE_URL) } coAnswers { + fallbackStartedAt.complete(scheduler.currentTime) + delay(150) + expected + } + + val result = YoutubeDirectLiveHlsStreamService(liveHlsService, fallbackService) + .getStreamInfo(YOUTUBE_URL) + + assertSame(expected, result) + assertEquals(100L, fallbackStartedAt.await()) + assertTrue(directCancelled.isCompleted) + } + + private fun liveResponse(): StreamResponse = mockk().also { + every { it.isLive } returns true + every { it.hlsUrl } returns HLS_URL + } + private companion object { const val VIDEO_ID = "GlzleRbo5E0" const val YOUTUBE_URL = "https://www.youtube.com/watch?v=$VIDEO_ID" From 7c66c32a02b8002b0c0e05824decd93d90b9bd04 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 14:32:15 +0000 Subject: [PATCH 093/137] perf: gate SABR startup on readable media --- .../server/routes/SabrPlaybackWindowTrackBuilder.kt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt index 52b337d4..4a35adda 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -61,13 +61,15 @@ internal class SabrPlaybackWindowTrackBuilder(private val sabrSessionStore: Sabr holder, format, seq, expectedStartMs, ) if (progressive != null) { + if (!progressive.hasReadableMedia) { + blockedRequest = mediaRequest + blockedBy = "${format.trackName()}:${format.itag}:$seq pending" + break + } seq = progressive.nextSequence coveredEndMs = progressive.coveredEndMs if (coveredEndMs >= goalEndMs) break - if (progressive.hasReadableMedia) continue - blockedRequest = SabrSegmentRequest.media(format, seq) - blockedBy = "${format.trackName()}:${format.itag}:$seq pending" - break + continue } blockedBy = "${format.trackName()}:${format.itag}:$seq pending" blockedRequest = mediaRequest @@ -140,5 +142,6 @@ internal data class TrackBuildResult( val coveredEndMs: Long, val atEnd: Boolean, ) { - fun covers(requiredEndMs: Long): Boolean = (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs + fun covers(requiredEndMs: Long): Boolean = + blockedBy == null && (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs } From edc5e827d6be7ebb5b17761e45b199f51d8f8429 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 14:46:09 +0000 Subject: [PATCH 094/137] chore: pause automatic validation workflows --- .github/workflows/ci.yml | 10 ++++++---- .github/workflows/coverage.yml | 6 ++++-- .github/workflows/openapi.yml | 10 ++++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff9408df..15cfdc27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,12 @@ name: CI on: - push: - branches: ["dev"] - pull_request: - branches: ["dev", "main"] + # Temporarily pause automatic CI while SABR startup is profiled. + # push: + # branches: ["dev"] + # pull_request: + # branches: ["dev", "main"] + workflow_dispatch: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c7f11e77..e997ba26 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,8 +1,10 @@ name: Coverage on: - pull_request: - branches: ["dev", "main"] + # Temporarily pause automatic test and coverage runs. + # pull_request: + # branches: ["dev", "main"] + workflow_dispatch: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index 1369962f..a44883f7 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -1,10 +1,12 @@ name: OpenAPI on: - push: - branches: ["dev"] - pull_request: - branches: ["dev", "main"] + # Temporarily pause automatic validation during SABR profiling. + # push: + # branches: ["dev"] + # pull_request: + # branches: ["dev", "main"] + workflow_dispatch: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true From aa115b8b124da8ae445fa2362dc1b395f82c2545 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 16:10:10 +0000 Subject: [PATCH 095/137] perf: keep subtitle lookup off SABR startup path --- .../dev/typetype/server/ExtractionServiceRegistry.kt | 1 + .../dev/typetype/server/services/PipePipeStreamService.kt | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 61411231..6a0d0656 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -102,6 +102,7 @@ internal class ExtractionServiceRegistry( youtubeSubtitleService, bilibiliRelatedService, sabrSessionStore::rememberExtractedInfo, + fetchSupplementalSubtitles = false, ) private val publicStreamService = YoutubePlayerClientStreamService( directPipePipeStreamService, diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index cf08ba67..d343acf1 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -37,6 +37,7 @@ class PipePipeStreamService( private val subtitleService: YouTubeSubtitleService, private val bilibiliRelatedService: BilibiliRelatedService, private val sabrInfoSink: (suspend (String, YoutubeSabrInfo) -> Unit)? = null, + private val fetchSupplementalSubtitles: Boolean = true, ) : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult = @@ -56,7 +57,11 @@ class PipePipeStreamService( rememberSabrInfo(streamInfo) streamInfo.setSponsorBlockSegments(segmentsDeferred.await()) val response = StreamAudioContractResolver.apply(streamInfo.toStreamResponse()) - val withSubtitles = if (response.subtitles.isEmpty() && service.serviceId == 0) { + val withSubtitles = if ( + fetchSupplementalSubtitles && + response.subtitles.isEmpty() && + service.serviceId == 0 + ) { val subtitles = try { subtitleService.fetchSubtitles(streamInfo.id) } catch (error: CancellationException) { From 3fc97a1fcca40ad719f31890d7c7573c6a6e7baa Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 18:11:23 +0000 Subject: [PATCH 096/137] perf: avoid blocking SABR extraction on redundant warmup --- .../services/SabrFallbackStreamService.kt | 25 ++++++++----------- .../server/services/SabrSessionStore.kt | 6 ++--- .../server/ExtractionServiceRegistry.kt | 2 +- .../server/services/PipePipeStreamService.kt | 4 +-- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt index bf01f9e3..6976a1bb 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrFallbackStreamService.kt @@ -2,33 +2,28 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope class SabrFallbackStreamService( private val delegate: StreamService, private val sessionStore: SabrSessionStore, private val tokenSessionClient: TypetypeTokenYoutubeSessionClient, ) : StreamService { - override suspend fun getStreamInfo(url: String): ExtractionResult = coroutineScope { - if (!isYoutubeUrl(url)) return@coroutineScope delegate.getStreamInfo(url) + override suspend fun getStreamInfo(url: String): ExtractionResult { + if (!isYoutubeUrl(url)) return delegate.getStreamInfo(url) val videoId = youtubeVideoId(url) - val prepared = videoId?.let { async { sessionStore.fetchInfo(it, cachedFirst = true) } } val result = delegate.getStreamInfo(url) val response = (result as? ExtractionResult.Success)?.data - if (response?.isLive == true) { - prepared?.cancel() - return@coroutineScope result + if (response?.isLive == true || response?.hasSabrStreams() == true) { + return result } if (response == null) { - if (result !is ExtractionResult.Failure || videoId == null) return@coroutineScope result - prepared?.await() - val session = tokenSessionClient.fetchPlaybackSession(videoId) ?: return@coroutineScope result - return@coroutineScope ExtractionResult.Success(session.toFallbackStreamResponse(videoId)) + if (result !is ExtractionResult.Failure || videoId == null) return result + val session = tokenSessionClient.fetchPlaybackSession(videoId) ?: return result + return ExtractionResult.Success(session.toFallbackStreamResponse(videoId)) } - if (response.hasSabrStreams() || videoId == null) return@coroutineScope result - val playable = prepared?.await() ?: return@coroutineScope result - ExtractionResult.Success(response.withSabrFallback(videoId, playable.info)) + if (videoId == null) return result + val playable = sessionStore.fetchInfo(videoId, cachedFirst = true) ?: return result + return ExtractionResult.Success(response.withSabrFallback(videoId, playable.info)) } } diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt index 40976939..702c3d1e 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrSessionStore.kt @@ -110,9 +110,7 @@ class SabrSessionStore( registry.lookupByItag(videoId, userId, itag) fun lookupByToken(videoId: String, token: String, itag: Int): SabrSessionHolder? = registry.lookupByToken(videoId, token, itag) - fun lookupByToken(videoId: String, token: String): SabrSessionHolder? = registry.lookupByToken(videoId, token) - fun lookupByToken(token: String): SabrSessionHolder? = registry.lookupByToken(token) suspend fun ensureWarmed(holder: SabrSessionHolder, maxPumps: Int = 8): Unit = pump.ensureWarmed(holder, maxPumps) @@ -149,7 +147,9 @@ class SabrSessionStore( cachedFirst: Boolean = false, isolatedPlayback: Boolean = false, ): SabrPreparedInfo? = infoFetcher.fetchInfo(videoId, startTimeMs, cachedFirst, isolatedPlayback) - + fun rememberExtractedInfoAsync(videoId: String, info: YoutubeSabrInfo): Unit { + scope.launch { infoFetcher.rememberExtractedInfo(videoId, info) } + } suspend fun rememberExtractedInfo(videoId: String, info: YoutubeSabrInfo): Unit = infoFetcher.rememberExtractedInfo(videoId, info) suspend fun rememberPreparedInfo(videoId: String, prepared: SabrPreparedInfo): Unit = infoFetcher.rememberPreparedInfo(videoId, prepared) diff --git a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt index 6a0d0656..1c983bba 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ExtractionServiceRegistry.kt @@ -101,7 +101,7 @@ internal class ExtractionServiceRegistry( cache, youtubeSubtitleService, bilibiliRelatedService, - sabrSessionStore::rememberExtractedInfo, + sabrSessionStore::rememberExtractedInfoAsync, fetchSupplementalSubtitles = false, ) private val publicStreamService = YoutubePlayerClientStreamService( diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index d343acf1..8ca271a9 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -36,7 +36,7 @@ class PipePipeStreamService( private val cache: CacheService, private val subtitleService: YouTubeSubtitleService, private val bilibiliRelatedService: BilibiliRelatedService, - private val sabrInfoSink: (suspend (String, YoutubeSabrInfo) -> Unit)? = null, + private val sabrInfoSink: ((String, YoutubeSabrInfo) -> Unit)? = null, private val fetchSupplementalSubtitles: Boolean = true, ) : StreamService { @@ -83,7 +83,7 @@ class PipePipeStreamService( ) } - private suspend fun rememberSabrInfo(streamInfo: StreamInfo): Unit { + private fun rememberSabrInfo(streamInfo: StreamInfo): Unit { val sink = sabrInfoSink ?: return val info = sequence { streamInfo.videoStreams.forEach { yield(it.deliveryMethodInfo) } From 3a159844cb9fbbd1255aeabf6accef5ea5cec12e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 22:34:01 +0000 Subject: [PATCH 097/137] perf: trace SABR startup across services --- .../dev/typetype/server/PlaybackTraceLog.kt | 18 ++++++++++++++++++ .../dev/typetype/server/RequestContext.kt | 9 ++++++--- .../server/routes/SabrPlaybackHandler.kt | 7 ++++++- .../kotlin/dev/typetype/server/AppMetrics.kt | 2 ++ .../typetype/server/RequestObservability.kt | 10 +++++++++- .../services/TypetypeTokenSabrTokenClient.kt | 19 ++++++++++++++++++- .../TypetypeTokenYoutubeSessionClient.kt | 17 ++++++++++++++++- 7 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt diff --git a/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt new file mode 100644 index 00000000..486c7ea0 --- /dev/null +++ b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt @@ -0,0 +1,18 @@ +package dev.typetype.server + +object PlaybackTraceLog { + private val logger = System.getLogger(PlaybackTraceLog::class.java.name) + + fun record(event: String, fields: String = "") { + record(currentPlaybackTraceId(), currentRequestId(), event, fields) + } + + fun record(traceId: String?, requestId: String?, event: String, fields: String = "") { + if (traceId == null) return + logger.log( + System.Logger.Level.INFO, + "playback_trace traceId={0} requestId={1} event={2} {3}", + traceId, requestId ?: "unknown", event, fields, + ) + } +} diff --git a/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt index ecbf335d..528824ed 100644 --- a/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/RequestContext.kt @@ -1,13 +1,16 @@ package dev.typetype.server -import kotlinx.coroutines.ThreadContextElement import kotlinx.coroutines.asContextElement +import kotlin.coroutines.CoroutineContext private val requestIdContext = ThreadLocal() +private val playbackTraceIdContext = ThreadLocal() const val REQUEST_ID_HEADER = "X-Request-ID" +const val PLAYBACK_TRACE_ID_HEADER = "X-Playback-Trace-ID" fun currentRequestId(): String? = requestIdContext.get() +fun currentPlaybackTraceId(): String? = playbackTraceIdContext.get() -fun requestContextElement(requestId: String?): ThreadContextElement = - requestIdContext.asContextElement(requestId) +fun requestContextElement(requestId: String?, playbackTraceId: String?): CoroutineContext = + requestIdContext.asContextElement(requestId) + playbackTraceIdContext.asContextElement(playbackTraceId) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 6c039d3d..65ceafcb 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -12,6 +12,7 @@ import dev.typetype.server.services.SabrPlaybackSegmentResult import dev.typetype.server.services.SabrPlaybackSessionService import dev.typetype.server.services.SabrPlaybackInfoResolver import dev.typetype.server.services.SabrSessionHolder +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.services.SabrSessionStore import dev.typetype.server.services.StreamService import dev.typetype.server.services.markServed @@ -39,10 +40,13 @@ internal class SabrPlaybackHandler( if (!validateAccess(call, videoId, access)) return val request = call.playbackRequest() val startTimeMs = request.effectiveStartTimeMs() + val infoStartedAt = System.nanoTime() val prepared = infoResolver.initial(access.userId, videoId, startTimeMs) - ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) + PlaybackTraceLog.record("sabr_info", "durationMs=${(System.nanoTime() - infoStartedAt) / 1_000_000} result=${if (prepared == null) "miss" else "ready"} startTimeMs=$startTimeMs") + prepared ?: return call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse("SABR probe failed")) val audio = selectAudio(call, prepared, request) ?: return val video = selectVideo(call, prepared, request) ?: return + val prepareStartedAt = System.nanoTime() val preparation = playbackService.prepare( videoId = videoId, userId = access.userId ?: videoId, @@ -53,6 +57,7 @@ internal class SabrPlaybackHandler( audioOnly = request.audioOnly, isLive = request.isLive, ) + PlaybackTraceLog.record("sabr_session_prepare", "durationMs=${(System.nanoTime() - prepareStartedAt) / 1_000_000} ready=${preparation.ready} startTimeMs=${preparation.startTimeMs}") preparation.holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) respondPrepared(call, preparation.holder, videoId, preparation.startTimeMs, preparation.ready) } diff --git a/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt b/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt index 0de33b49..75cb8467 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/AppMetrics.kt @@ -51,6 +51,8 @@ object AppMetrics { } fun metricPath(path: String): String = when { + path.startsWith("/api/sabr/playback/") -> "/api/sabr/playback/{sessionId}" + path.startsWith("/sabr/playback/") -> "/sabr/playback/{sessionId}" path.startsWith("/downloader/jobs/") && path.endsWith("/events") -> "/downloader/jobs/{id}/events" path.startsWith("/downloader/jobs/") && path.endsWith("/artifact") -> "/downloader/jobs/{id}/artifact" path.startsWith("/downloader/jobs/") && path.endsWith("/cancel") -> "/downloader/jobs/{id}/cancel" diff --git a/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt b/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt index e7e2d3ca..f00f0158 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/RequestObservability.kt @@ -25,13 +25,18 @@ fun Application.installRequestObservability() { intercept(ApplicationCallPipeline.Setup) { val applicationCall = context val requestId = resolveRequestId(applicationCall.request.headers[REQUEST_ID_HEADER]) + val traceId = resolvePlaybackTraceId(applicationCall.request.headers[PLAYBACK_TRACE_ID_HEADER]) applicationCall.attributes.put(requestIdAttribute, requestId) applicationCall.attributes.put(requestStartNanosAttribute, System.nanoTime()) applicationCall.response.headers.append(REQUEST_ID_HEADER, requestId, safeOnly = false) - withContext(requestContextElement(requestId)) { + traceId?.let { applicationCall.response.headers.append(PLAYBACK_TRACE_ID_HEADER, it, safeOnly = false) } + withContext(requestContextElement(requestId, traceId)) { try { proceed() } finally { + traceId?.let { + PlaybackTraceLog.record(it, requestId, "http_complete", "method=${applicationCall.request.httpMethod.value} route=${metricPath(applicationCall.request.path())} status=${applicationCall.response.status()?.value ?: 0} durationMs=${applicationCall.requestDurationMs()}") + } AppMetrics.record(applicationCall) } } @@ -49,5 +54,8 @@ fun requestLogLine(call: ApplicationCall): String = listOf( private fun resolveRequestId(raw: String?): String = raw?.takeIf { requestIdRegex.matches(it) } ?: UUID.randomUUID().toString() +private fun resolvePlaybackTraceId(raw: String?): String? = + raw?.takeIf { requestIdRegex.matches(it) } + private fun ApplicationCall.attributeOrNull(key: AttributeKey): T? = if (attributes.contains(key)) attributes[key] else null diff --git a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt index 507229c1..b4b1b1d3 100644 --- a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenSabrTokenClient.kt @@ -5,6 +5,11 @@ import okhttp3.MediaType.Companion.toMediaType import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject +import dev.typetype.server.PLAYBACK_TRACE_ID_HEADER +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.currentPlaybackTraceId +import dev.typetype.server.currentRequestId import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -34,9 +39,15 @@ class TypetypeTokenSabrTokenClient( val request = Request.Builder() .url("${tokenServiceUrl.trimEnd('/')}/potoken/session") .post(body) + .apply { + currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { header(PLAYBACK_TRACE_ID_HEADER, it) } + } .build() + val startedAt = System.nanoTime() return try { client.newCall(request).execute().use { response -> + PlaybackTraceLog.record("token_http", "endpoint=potoken_session status=${response.code} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (!response.isSuccessful) return null SabrTokenBundle.fromSessionResponse(videoId, sessionBinding, JSONObject(response.body.string())) } @@ -56,8 +67,14 @@ class TypetypeTokenSabrTokenClient( ): SabrTokenBundle? { val url = buildUrl(binding, forceRefresh, refreshVideo) val target = if (logIdentifier) " for $binding" else " for session binding" + val startedAt = System.nanoTime() + val request = Request.Builder().url(url).get().apply { + currentRequestId()?.let { header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { header(PLAYBACK_TRACE_ID_HEADER, it) } + }.build() return try { - client.newCall(Request.Builder().url(url).get().build()).execute().use { resp -> + client.newCall(request).execute().use { resp -> + PlaybackTraceLog.record("token_http", "endpoint=potoken status=${resp.code} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (!resp.isSuccessful) { System.err.println("[TypetypeTokenSabrTokenClient] /potoken HTTP ${resp.code}$target") return null diff --git a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt index fd4422fa..fa465918 100644 --- a/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt +++ b/server-token-gateway/src/main/kotlin/dev/typetype/server/services/TypetypeTokenYoutubeSessionClient.kt @@ -11,6 +11,11 @@ import okhttp3.Response import org.json.JSONObject import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import dev.typetype.server.sabr.SabrAdapter +import dev.typetype.server.PLAYBACK_TRACE_ID_HEADER +import dev.typetype.server.REQUEST_ID_HEADER +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.currentPlaybackTraceId +import dev.typetype.server.currentRequestId import dev.typetype.server.sabr.YoutubeSabrClientProfile import dev.typetype.server.sabr.YoutubeSabrInfo import java.io.IOException @@ -67,17 +72,27 @@ class TypetypeTokenYoutubeSessionClient( val url = "${tokenServiceUrl.trimEnd('/')}/youtube/sabr/session?videoId=$encodedVideoId&client=MWEB" + if (isolated) "&isolated=true" else "" return suspendCancellableCoroutine { continuation -> - val call = client.newCall(Request.Builder().url(url).get().build()) + val requestBuilder = Request.Builder().url(url).get() + currentRequestId()?.let { requestBuilder.header(REQUEST_ID_HEADER, it) } + currentPlaybackTraceId()?.let { requestBuilder.header(PLAYBACK_TRACE_ID_HEADER, it) } + val traceId = currentPlaybackTraceId() + val requestId = currentRequestId() + val startedAt = System.nanoTime() + PlaybackTraceLog.record("token_session_start", "operation=youtube_sabr_session isolated=$isolated") + val call = client.newCall(requestBuilder.build()) continuation.invokeOnCancellation { call.cancel() } call.enqueue(object : Callback { override fun onFailure(call: Call, e: IOException) { + PlaybackTraceLog.record(traceId, requestId, "token_session_complete", "status=network_error durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (continuation.isActive) continuation.resume(null) } override fun onResponse(call: Call, response: Response) { + val status = response.code val session = response.use { if (!it.isSuccessful) null else runCatching { JSONObject(it.body.string()) }.getOrNull() } + PlaybackTraceLog.record(traceId, requestId, "token_session_complete", "status=$status result=${if (session == null) "empty" else "ok"} durationMs=${(System.nanoTime() - startedAt) / 1_000_000}") if (continuation.isActive) continuation.resume(session) } }) From 7be28933d82d802bd10458565bab7f366d477f6b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 22:34:11 +0000 Subject: [PATCH 098/137] docs: document playback startup tracing --- PERF.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 PERF.md diff --git a/PERF.md b/PERF.md new file mode 100644 index 00000000..1474562d --- /dev/null +++ b/PERF.md @@ -0,0 +1,38 @@ +# Playback Startup Performance + +## Target + +The target is less than 3 seconds from selecting a video on `beta.typetype.video` to its first rendered video frame. The browser trace's `playback.first_frame` event measures this interval from the click trace; a direct watch-page load starts at route entry instead. + +This target is not yet demonstrated. A successful build or fast API response is not a playback result. Validate against the deployed Beta build in Chromium and Firefox, using the same video and recording cold and warm starts separately. Do not infer a pass from a cached thumbnail or `loadedmetadata`; require `first_frame`. + +## PipePipe Reference + +The checked upstream PipePipe Client is commit `bf92523449024b349ff6dc1a3ea4f6f68327bb23`, version `5.4.0-beta2`; PipePipeExtractor is `df983df0e5a72f022f9bd2f066a42117861e4455`. PipePipe's production `PlaybackStartupTrace` marks detail click, stream resolution, player setup, SABR source setup, and first frame. Its Android instrumentation benchmark and request dumper provide deeper test-only Media3 and SABR measurements. + +TypeType runs in a browser with separate Server and Token processes, so the implementation correlates those boundaries with an opt-in `X-Playback-Trace-ID`. It is TypeType instrumentation, not a PipePipe code change or a claim that the two players have equivalent startup time. + +## Enable Browser Trace + +Before starting a video, open the browser developer console on Beta and run: + +```js +localStorage.setItem("typetype-debug-console", "1"); +location.reload(); +``` + +Start a video from its card. In the console, collect all `[typetype] playback.*` records for the `traceId` emitted by `playback.trace_start`, through `playback.first_frame`. Remove the setting with `localStorage.removeItem("typetype-debug-console")` to stop tracing. Browser records are kept in session storage for up to 30 minutes, capped at 800 playback entries. + +## Correlated Logs + +The same trace ID is sent on selected stream, comments, subtitle, SABR prewarm, and SABR MSE requests. Server echoes it and forwards it, with the request ID, to Token. Filter Server and Token runtime logs for `[playback_trace]` and the same `traceId`. + +Browser events include API start/end/status/duration, video readiness and playback events, the first rendered frame, same-origin SABR resource timing and transfer sizes, and supported long-task observations. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. + +Compare the browser's full API duration with the Server request duration and the nested Token phases. This separates browser/MSE waiting, Server work, and Token work without treating a fast response header as a rendered frame. + +Detailed records are emitted only for requests carrying a valid trace ID. They do not include bearer tokens, cookies, PO-token values, session bindings, request bodies, or signed query strings. Subtitle timing is observational; this harness does not make subtitle loading a prerequisite for video playback. + +## Acceptance + +A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. Keep the matching browser events and Server/Token records by trace ID with the video and whether the run was cold or warm. Until those deployed measurements exist, the startup target remains unverified. From 092bd14e11a228f220bb6bbbfb6acb4fdcf86cf1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 24 Sep 2026 22:50:50 +0000 Subject: [PATCH 099/137] perf: log selected SABR startup formats --- .../kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 65ceafcb..8980afec 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -57,7 +57,7 @@ internal class SabrPlaybackHandler( audioOnly = request.audioOnly, isLive = request.isLive, ) - PlaybackTraceLog.record("sabr_session_prepare", "durationMs=${(System.nanoTime() - prepareStartedAt) / 1_000_000} ready=${preparation.ready} startTimeMs=${preparation.startTimeMs}") + PlaybackTraceLog.record("sabr_session_prepare", "durationMs=${(System.nanoTime() - prepareStartedAt) / 1_000_000} ready=${preparation.ready} videoItag=${video.itag} audioItag=${audio.itag} audioTrackId=${audio.audioTrackId} audioOnly=${request.audioOnly} isLive=${request.isLive} startTimeMs=${preparation.startTimeMs}") preparation.holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) respondPrepared(call, preparation.holder, videoId, preparation.startTimeMs, preparation.ready) } From b62ad2a4cbe7d24ab1d0fb482dcfb59bae58eb2c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 00:14:00 +0000 Subject: [PATCH 100/137] perf: reuse prewarmed SABR playback sessions --- openapi.yaml | 1 + openapi/components/sabr-playback.yaml | 17 ++++ openapi/paths/sabr-playback.yaml | 46 +++++++++++ .../server/routes/SabrPlaybackHandler.kt | 36 ++++----- .../routes/SabrPlaybackPrewarmHandoffs.kt | 79 +++++++++++++++++++ .../server/routes/SabrPlaybackRequests.kt | 26 ++++++ 6 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt diff --git a/openapi.yaml b/openapi.yaml index dc141567..71741dd6 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -32,6 +32,7 @@ paths: /proxy: { $ref: ./openapi/paths/proxy.yaml#/Proxy } /media/{handle}: { $ref: ./openapi/paths/media.yaml#/ProviderMedia } /sabr/download/{videoId}: { $ref: ./openapi/paths/sabr-download.yaml#/SabrDownload } + /sabr/playback/{videoId}: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackCreate } /sabr/playback/{sessionId}/position: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPosition } /sabr/playback/{sessionId}/window: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackWindow } /sabr/playback/{sessionId}/prefetch: { $ref: ./openapi/paths/sabr-playback.yaml#/SabrPlaybackPrefetch } diff --git a/openapi/components/sabr-playback.yaml b/openapi/components/sabr-playback.yaml index 6a46b981..382e84ee 100644 --- a/openapi/components/sabr-playback.yaml +++ b/openapi/components/sabr-playback.yaml @@ -1,3 +1,20 @@ +SabrPlaybackCreateResponse: + type: object + required: [sessionId, videoId, videoItag, audioItag, startTimeMs, generation, ready, status] + additionalProperties: true + properties: + sessionId: { type: string } + videoId: { type: string } + manifestUrl: { type: string, nullable: true } + videoItag: { type: integer, minimum: 1 } + audioItag: { type: integer, minimum: 1 } + audioTrackId: { type: string, nullable: true } + startTimeMs: { type: integer, format: int64, minimum: 0 } + generation: { type: integer, format: int64, minimum: 0 } + ready: { type: boolean } + status: { type: string } + retryAfterMs: { type: integer, format: int64, nullable: true } + live: { type: object, nullable: true, additionalProperties: true } SabrPlaybackBufferedRange: type: object required: [itag, startMs, endMs] diff --git a/openapi/paths/sabr-playback.yaml b/openapi/paths/sabr-playback.yaml index aa281000..00e21ceb 100644 --- a/openapi/paths/sabr-playback.yaml +++ b/openapi/paths/sabr-playback.yaml @@ -1,3 +1,49 @@ +SabrPlaybackCreate: + post: + tags: [playback] + summary: Create a SABR session or transfer a matching prewarmed session + parameters: + - name: videoId + in: path + required: true + schema: { type: string } + - name: videoItag + in: query + schema: { type: integer, minimum: 1 } + - name: audioItag + in: query + schema: { type: integer, minimum: 1 } + - name: audioTrackId + in: query + schema: { type: string } + - name: startTimeMs + in: query + schema: { type: integer, format: int64, minimum: 0, default: 0 } + - name: audioOnly + in: query + schema: { type: boolean, default: false } + - name: isLive + in: query + schema: { type: boolean, default: false } + - name: prewarm + in: query + description: Keep this session briefly for the next matching creation by the same account. + schema: { type: boolean, default: false } + responses: + '200': + description: Playback session created or transferred. + content: + application/json: + schema: + $ref: ../components/sabr-playback.yaml#/SabrPlaybackCreateResponse + '202': + description: Playback session is still preparing. + content: + application/json: + schema: + $ref: ../components/sabr-playback.yaml#/SabrPlaybackCreateResponse + '422': + $ref: ../components/common.yaml#/JsonError SabrPlaybackPosition: post: tags: [playback] diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt index 8980afec..87ec3e89 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt @@ -18,7 +18,6 @@ import dev.typetype.server.services.StreamService import dev.typetype.server.services.markServed import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall -import io.ktor.server.request.receive import io.ktor.server.response.respond import dev.typetype.server.sabr.YoutubeSabrFormat @@ -34,12 +33,23 @@ internal class SabrPlaybackHandler( private val playbackService = SabrPlaybackSessionService(sabrSessionStore) private val infoResolver = SabrPlaybackInfoResolver(sabrSessionStore, authenticatedSabrInfoService) private val accessValidator = SabrPlaybackAccessValidator(streamService, youtubeSessionStreamInfo) + private val prewarmHandoffs = SabrPlaybackPrewarmHandoffs() suspend fun create(call: ApplicationCall, videoId: String) { val access = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService) ?: return if (!validateAccess(call, videoId, access)) return val request = call.playbackRequest() val startTimeMs = request.effectiveStartTimeMs() + val isPrewarm = call.request.queryParameters["prewarm"]?.toBooleanStrictOrNull() == true + val handoffKey = prewarmHandoffs.key(access.userId, videoId, request, startTimeMs) + if (!isPrewarm) { + val handoff = handoffKey?.let(prewarmHandoffs::take) + val existing = handoff?.let { playbackService.lookup(it.sessionToken) } + if (existing != null && existing.terminalFailure() == null) { + PlaybackTraceLog.record("sabr_prewarm_handoff", "result=hit videoItag=${existing.videoFormat.itag} audioItag=${existing.audioFormat.itag} audioTrackId=${existing.audioFormat.audioTrackId} startTimeMs=${handoff.startTimeMs} ready=${handoff.ready}") + return respondPrepared(call, existing, videoId, handoff.startTimeMs, handoff.ready) + } + } val infoStartedAt = System.nanoTime() val prepared = infoResolver.initial(access.userId, videoId, startTimeMs) PlaybackTraceLog.record("sabr_info", "durationMs=${(System.nanoTime() - infoStartedAt) / 1_000_000} result=${if (prepared == null) "miss" else "ready"} startTimeMs=$startTimeMs") @@ -59,6 +69,10 @@ internal class SabrPlaybackHandler( ) PlaybackTraceLog.record("sabr_session_prepare", "durationMs=${(System.nanoTime() - prepareStartedAt) / 1_000_000} ready=${preparation.ready} videoItag=${video.itag} audioItag=${audio.itag} audioTrackId=${audio.audioTrackId} audioOnly=${request.audioOnly} isLive=${request.isLive} startTimeMs=${preparation.startTimeMs}") preparation.holder.setActiveTracks(videoActive = !request.audioOnly, audioActive = true) + if (isPrewarm && handoffKey != null && preparation.holder.terminalFailure() == null) { + prewarmHandoffs.remember(handoffKey, SabrPlaybackPrewarmSession(preparation.holder.sessionToken, preparation.startTimeMs, preparation.ready)) + PlaybackTraceLog.record("sabr_prewarm_handoff", "result=stored videoItag=${video.itag} audioItag=${audio.itag} audioTrackId=${audio.audioTrackId} startTimeMs=$startTimeMs") + } respondPrepared(call, preparation.holder, videoId, preparation.startTimeMs, preparation.ready) } @@ -160,26 +174,6 @@ internal class SabrPlaybackHandler( } } - private suspend fun ApplicationCall.playbackRequest(): SabrPlaybackRequest { - val body = runCatching { receive() }.getOrNull() - return SabrPlaybackRequest( - videoItag = body?.videoItag ?: request.queryParameters["videoItag"]?.toIntOrNull(), - audioItag = body?.audioItag ?: request.queryParameters["audioItag"]?.toIntOrNull(), - audioTrackId = body?.audioTrackId ?: request.queryParameters["audioTrackId"], - startTimeMs = body?.startTimeMs ?: request.queryParameters["startTimeMs"]?.toLongOrNull(), - playerTimeMs = body?.playerTimeMs ?: request.queryParameters["playerTimeMs"]?.toLongOrNull(), - audioOnly = body?.audioOnly ?: request.queryParameters["audioOnly"]?.toBooleanStrictOrNull() ?: false, - isLive = body?.isLive ?: request.queryParameters["isLive"]?.toBooleanStrictOrNull() ?: false, - ) - } - - private fun SabrPlaybackRequest.effectiveStartTimeMs(): Long = - (playerTimeMs ?: startTimeMs ?: 0L).coerceAtLeast(0L) - - private fun SabrPlaybackRequest.keepsCurrentFormats(holder: SabrSessionHolder): Boolean = - (videoItag == null || videoItag == holder.videoFormat.itag) && - (audioItag == null || audioItag == holder.audioFormat.itag) && - (audioTrackId.isNullOrBlank() || audioTrackId == holder.audioFormat.audioTrackId) private suspend fun selectAudio( call: ApplicationCall, diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt new file mode 100644 index 00000000..97fea167 --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackPrewarmHandoffs.kt @@ -0,0 +1,79 @@ +package dev.typetype.server.routes + +import java.util.LinkedHashMap + +internal data class SabrPlaybackPrewarmKey( + val userId: String, + val videoId: String, + val videoItag: Int, + val audioItag: Int, + val audioTrackId: String?, + val startTimeMs: Long, + val audioOnly: Boolean, + val isLive: Boolean, +) + +internal data class SabrPlaybackPrewarmSession( + val sessionToken: String, + val startTimeMs: Long, + val ready: Boolean, +) + +internal class SabrPlaybackPrewarmHandoffs( + private val maxEntries: Int = MAX_ENTRIES, + private val ttlNanos: Long = TTL_NANOS, +) { + private data class Entry(val session: SabrPlaybackPrewarmSession, val expiresAtNanos: Long) + + private val entries = LinkedHashMap() + + fun key( + userId: String?, + videoId: String, + request: SabrPlaybackRequest, + startTimeMs: Long, + ): SabrPlaybackPrewarmKey? { + val scope = userId?.takeIf { it.isNotBlank() } ?: return null + val videoItag = request.videoItag ?: return null + val audioItag = request.audioItag ?: return null + return SabrPlaybackPrewarmKey( + userId = scope, + videoId = videoId, + videoItag = videoItag, + audioItag = audioItag, + audioTrackId = request.audioTrackId, + startTimeMs = startTimeMs, + audioOnly = request.audioOnly, + isLive = request.isLive, + ) + } + + @Synchronized + fun remember(key: SabrPlaybackPrewarmKey, session: SabrPlaybackPrewarmSession) { + val now = System.nanoTime() + evictExpired(now) + entries[key] = Entry(session, now + ttlNanos) + while (entries.size > maxEntries) { + entries.remove(entries.keys.first()) + } + } + + @Synchronized + fun take(key: SabrPlaybackPrewarmKey): SabrPlaybackPrewarmSession? { + val now = System.nanoTime() + val entry = entries.remove(key) ?: return null + return entry.session.takeUnless { now - entry.expiresAtNanos >= 0L } + } + + private fun evictExpired(now: Long) { + val iterator = entries.values.iterator() + while (iterator.hasNext()) { + if (now - iterator.next().expiresAtNanos >= 0L) iterator.remove() + } + } + + private companion object { + const val MAX_ENTRIES = 64 + const val TTL_NANOS = 60_000_000_000L + } +} diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt new file mode 100644 index 00000000..3c3b658b --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackRequests.kt @@ -0,0 +1,26 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SabrSessionHolder +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive + +internal suspend fun ApplicationCall.playbackRequest(): SabrPlaybackRequest { + val body = runCatching { receive() }.getOrNull() + return SabrPlaybackRequest( + videoItag = body?.videoItag ?: request.queryParameters["videoItag"]?.toIntOrNull(), + audioItag = body?.audioItag ?: request.queryParameters["audioItag"]?.toIntOrNull(), + audioTrackId = body?.audioTrackId ?: request.queryParameters["audioTrackId"], + startTimeMs = body?.startTimeMs ?: request.queryParameters["startTimeMs"]?.toLongOrNull(), + playerTimeMs = body?.playerTimeMs ?: request.queryParameters["playerTimeMs"]?.toLongOrNull(), + audioOnly = body?.audioOnly ?: request.queryParameters["audioOnly"]?.toBooleanStrictOrNull() ?: false, + isLive = body?.isLive ?: request.queryParameters["isLive"]?.toBooleanStrictOrNull() ?: false, + ) +} + +internal fun SabrPlaybackRequest.effectiveStartTimeMs(): Long = + (playerTimeMs ?: startTimeMs ?: 0L).coerceAtLeast(0L) + +internal fun SabrPlaybackRequest.keepsCurrentFormats(holder: SabrSessionHolder): Boolean = + (videoItag == null || videoItag == holder.videoFormat.itag) && + (audioItag == null || audioItag == holder.audioFormat.itag) && + (audioTrackId.isNullOrBlank() || audioTrackId == holder.audioFormat.audioTrackId) From d596c984889476a208298c643a37bb3c1b2c6643 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 00:55:20 +0000 Subject: [PATCH 101/137] fix: route playback trace through slf4j --- server-core/build.gradle.kts | 1 + .../kotlin/dev/typetype/server/PlaybackTraceLog.kt | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server-core/build.gradle.kts b/server-core/build.gradle.kts index b4b4830b..4df09914 100644 --- a/server-core/build.gradle.kts +++ b/server-core/build.gradle.kts @@ -7,6 +7,7 @@ plugins { dependencies { api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") + implementation("org.slf4j:slf4j-api:2.0.18") testImplementation(testFixtures(project(":server-core"))) testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") diff --git a/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt index 486c7ea0..de9dd35d 100644 --- a/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/PlaybackTraceLog.kt @@ -1,7 +1,9 @@ package dev.typetype.server +import org.slf4j.LoggerFactory + object PlaybackTraceLog { - private val logger = System.getLogger(PlaybackTraceLog::class.java.name) + private val logger = LoggerFactory.getLogger(PlaybackTraceLog::class.java) fun record(event: String, fields: String = "") { record(currentPlaybackTraceId(), currentRequestId(), event, fields) @@ -9,10 +11,6 @@ object PlaybackTraceLog { fun record(traceId: String?, requestId: String?, event: String, fields: String = "") { if (traceId == null) return - logger.log( - System.Logger.Level.INFO, - "playback_trace traceId={0} requestId={1} event={2} {3}", - traceId, requestId ?: "unknown", event, fields, - ) + logger.info("playback_trace traceId={} requestId={} event={} {}", traceId, requestId ?: "unknown", event, fields) } } From 2562929266dc90e7cdf2c6cff1f98be39ecdc859 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 01:24:32 +0000 Subject: [PATCH 102/137] docs: record Beta playback startup measurements --- PERF.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/PERF.md b/PERF.md index 1474562d..9254a96c 100644 --- a/PERF.md +++ b/PERF.md @@ -4,7 +4,7 @@ The target is less than 3 seconds from selecting a video on `beta.typetype.video` to its first rendered video frame. The browser trace's `playback.first_frame` event measures this interval from the click trace; a direct watch-page load starts at route entry instead. -This target is not yet demonstrated. A successful build or fast API response is not a playback result. Validate against the deployed Beta build in Chromium and Firefox, using the same video and recording cold and warm starts separately. Do not infer a pass from a cached thumbnail or `loadedmetadata`; require `first_frame`. +Two prewarmed Beta browser samples below are under 3 seconds. This does not establish cold-start behavior or a percentile target. Do not infer a pass from a cached thumbnail or `loadedmetadata`; require `first_frame`. ## PipePipe Reference @@ -33,6 +33,23 @@ Compare the browser's full API duration with the Server request duration and the Detailed records are emitted only for requests carrying a valid trace ID. They do not include bearer tokens, cookies, PO-token values, session bindings, request bodies, or signed query strings. Subtitle timing is observational; this harness does not make subtitle loading a prerequisite for video playback. +## Measurements (2026-09-25) + +The public Beta instance reported Server revision `d596c984`. With the same video and fresh browser contexts, both runs used the SABR prewarm path: + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,731 ms | 1,759 ms | 3.94 s | Pass | +| Firefox | 2,114 ms | 2,159 ms | 4.03 s | Pass | + +Both rendered a 1920x1080 frame, continued playing without a media error, and had about 5.34 seconds buffered at `loadeddata`, growing to about 10.68 seconds. Two text tracks were attached and zero were visible at first frame in both runs. This demonstrates that subtitles did not block these starts; it does not establish behavior for every subtitle format or cold start. + +These are two prewarmed samples, not cold-start measurements or a percentile. Browser SABR stream-info calls took 369 ms in Chromium and 50 ms in Firefox; the measured SABR create requests took 94/10 ms and 36/26 ms respectively (prewarm/consumer). Segment requests completed in 25-38 ms in Chromium and 26-66 ms in Firefox. + +A separate cold direct Stack Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. + +The direct Stack request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Stack-local Server or Token container logs, so public-runtime log correlation is not yet verified. Keep this distinction when comparing browser timings with backend phases. + ## Acceptance -A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. Keep the matching browser events and Server/Token records by trace ID with the video and whether the run was cold or warm. Until those deployed measurements exist, the startup target remains unverified. +A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The two prewarmed samples above pass. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. From 46ba92ebdc241a92ebf4ae91702d149da92dd40d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 02:24:16 +0000 Subject: [PATCH 103/137] perf: trace playback extraction startup phases --- PERF.md | 12 ++- .../typetype/server/routes/StreamRoutes.kt | 27 ++++++ .../services/SabrBootstrapStreamService.kt | 19 +++- .../server/services/SabrInfoFetcher.kt | 8 +- .../server/services/PipePipeStreamService.kt | 89 ++++++++++++++----- 5 files changed, 125 insertions(+), 30 deletions(-) diff --git a/PERF.md b/PERF.md index 9254a96c..f151f70c 100644 --- a/PERF.md +++ b/PERF.md @@ -8,9 +8,9 @@ Two prewarmed Beta browser samples below are under 3 seconds. This does not esta ## PipePipe Reference -The checked upstream PipePipe Client is commit `bf92523449024b349ff6dc1a3ea4f6f68327bb23`, version `5.4.0-beta2`; PipePipeExtractor is `df983df0e5a72f022f9bd2f066a42117861e4455`. PipePipe's production `PlaybackStartupTrace` marks detail click, stream resolution, player setup, SABR source setup, and first frame. Its Android instrumentation benchmark and request dumper provide deeper test-only Media3 and SABR measurements. +The checked upstreams are the official `InfinityLoop1308` repositories: [PipePipe Client `c2a166f7`](https://github.com/InfinityLoop1308/PipePipeClient/commit/c2a166f7df05e5ace4c80fb3e3fd27702cfc3e56), declared version `5.4.0`, and [PipePipeExtractor `c68e10e2`](https://github.com/InfinityLoop1308/PipePipeExtractor/commit/c68e10e2e97495877832d8df6cbac55478083019), version `5.4.0`. In Client, `SabrDashMediaSource` calls `SabrMediaBridge.prepareTimelines`; the bridge requests initial audio and video timelines together and retains returned media. Its `PlaybackStartupTrace` marks production player phases, while `SabrPlaybackSmokeTest` and `YoutubePlaybackBenchmarkTest` provide Android test instrumentation. Those harnesses do not measure TypeType's browser or Beta backend. Extractor owns the UMP/SABR session and request logic; its [SABR research log](https://github.com/InfinityLoop1308/PipePipeExtractor/issues/66) is issue-based rather than a standalone current guide. -TypeType runs in a browser with separate Server and Token processes, so the implementation correlates those boundaries with an opt-in `X-Playback-Trace-ID`. It is TypeType instrumentation, not a PipePipe code change or a claim that the two players have equivalent startup time. +TypeType's relevant difference is its separate HTTP orchestration: browser requests for stream metadata/bootstrap, Server-side Token resolution, a prewarm-to-consumer session handoff, then browser MSE requests for init and segments. The PipePipe sources are a protocol and instrumentation reference, not evidence that TypeType has the same startup profile or that an upstream change is needed. ## Enable Browser Trace @@ -46,10 +46,14 @@ Both rendered a 1920x1080 frame, continued playing without a media error, and ha These are two prewarmed samples, not cold-start measurements or a percentile. Browser SABR stream-info calls took 369 ms in Chromium and 50 ms in Firefox; the measured SABR create requests took 94/10 ms and 36/26 ms respectively (prewarm/consumer). Segment requests completed in 25-38 ms in Chromium and 26-66 ms in Firefox. +A later card-click sample on video `ynKvUYpu1Qw` measured 1,892 ms to first frame in Chromium and 1,913 ms in Firefox. Playback started at 1,908/1,941 ms, continued for 3.94/4.01 seconds, and rendered 1920x1080 without a media error. Chromium's stream-info call took 1,291 ms and bootstrap took 172 ms; session creation took 318 ms for the consumer and 139 ms for prewarm. Firefox ran after Chromium had used the same video: stream-info took 43 ms, session calls 29-30 ms, and init calls 26-27 ms. No text tracks were attached at first frame. Firefox was cache-warm, and Chromium's origin-side cache state was not measured, so these results do not prove a cold-start pass. + +The TypeType SABR extractor configuration disables supplemental subtitle lookup. This shows subtitles did not gate this start, not that every subtitle path is non-blocking. + A separate cold direct Stack Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. -The direct Stack request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Stack-local Server or Token container logs, so public-runtime log correlation is not yet verified. Keep this distinction when comparing browser timings with backend phases. +The direct Stack request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Stack-local Server or Token container logs, so public-runtime log correlation is not yet verified. The new trace-gated Server events split provider setup, page fetch, stream extraction, SponsorBlock, supplemental subtitles, authenticated resolution, SABR bootstrap metadata, and format preparation. Verify they appear with the browser trace ID after the change reaches Beta before attributing public latency to a backend phase. ## Acceptance -A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The two prewarmed samples above pass. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. +A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The four browser samples above pass individually. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index c5cd2adf..c96da3af 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -1,5 +1,6 @@ package dev.typetype.server.routes +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.models.ErrorResponse import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse @@ -75,6 +76,8 @@ private fun Route.streamRoute( ErrorResponse("URL does not match stream endpoint", "provider_mismatch"), ) } + PlaybackTraceLog.record("stream_route_start", "deliveryMode=$deliveryMode") + val accessStartedAt = System.nanoTime() val access = call.accessProfileOrRespond( dependencies.authService, dependencies.accessControlService, @@ -84,14 +87,38 @@ private fun Route.streamRoute( val blockedProfile = access.userId ?.let { dependencies.blockedService?.profileFor(it) } ?: BlockedContentProfile.empty + PlaybackTraceLog.record( + "stream_route_access", + "durationMs=${(System.nanoTime() - accessStartedAt) / 1_000_000} guest=${access.userId == null}", + ) if (blockedProfile.blocksVideo(url)) { return@get call.respond( HttpStatusCode.Forbidden, ErrorResponse("Video is blocked", "content_blocked"), ) } + val providerStartedAt = System.nanoTime() val publicResult = streamService.getStreamInfo(url) + val providerResult = when (publicResult) { + is ExtractionResult.Success -> "success" + is ExtractionResult.Failure -> "failure" + is ExtractionResult.BadRequest -> "bad_request" + } + PlaybackTraceLog.record( + "stream_provider_info", + "durationMs=${(System.nanoTime() - providerStartedAt) / 1_000_000} result=$providerResult", + ) + val resolutionStartedAt = System.nanoTime() val resolution = resolveStreamInfo(url, deliveryMode, access.userId, publicResult, dependencies) + val resolutionResult = when (resolution.result) { + is ExtractionResult.Success -> "success" + is ExtractionResult.Failure -> "failure" + is ExtractionResult.BadRequest -> "bad_request" + } + PlaybackTraceLog.record( + "stream_resolution", + "durationMs=${(System.nanoTime() - resolutionStartedAt) / 1_000_000} result=$resolutionResult authenticated=${resolution.authenticated}", + ) when (val result = resolution.result) { is ExtractionResult.Success -> { if (!accessProfile.allowsUploader(result.data.uploaderUrl, result.data.uploaderName)) { diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt index d7b3d746..230a8e4f 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrBootstrapStreamService.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.StreamResponse @@ -11,11 +12,23 @@ class SabrBootstrapStreamService( override suspend fun getStreamInfo(url: String): ExtractionResult { val videoId = youtubeVideoId(url) ?: return ExtractionResult.BadRequest("Invalid YouTube URL") - val metadata = tokenSessionClient.fetchPlaybackSession(videoId) + val metadataStartedAt = System.nanoTime() + val metadataResult = tokenSessionClient.fetchPlaybackSession(videoId) + PlaybackTraceLog.record( + "sabr_bootstrap_metadata", + "durationMs=${(System.nanoTime() - metadataStartedAt) / 1_000_000} result=${if (metadataResult == null) "miss" else "ready"}", + ) + val metadata = metadataResult ?: return ExtractionResult.Failure("SABR bootstrap metadata unavailable") if (metadata.isLive) return liveHlsStreamService.getStreamInfo(url) - val prepared = metadata.preparedSabrInfo() - ?: sessionStore.fetchInfo(videoId, cachedFirst = true) + val formatsStartedAt = System.nanoTime() + val fromSession = metadata.preparedSabrInfo() + val formatsResult = fromSession ?: sessionStore.fetchInfo(videoId, cachedFirst = true) + PlaybackTraceLog.record( + "sabr_bootstrap_formats", + "durationMs=${(System.nanoTime() - formatsStartedAt) / 1_000_000} source=${if (fromSession != null) "session" else "cache_or_probe"} result=${if (formatsResult == null) "miss" else "ready"}", + ) + val prepared = formatsResult ?: return ExtractionResult.Failure("SABR playback formats unavailable") sessionStore.rememberPreparedInfo(videoId, prepared) return ExtractionResult.Success(metadata.toFallbackStreamResponse(videoId)) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt index c81b22fc..81543c2d 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrInfoFetcher.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.cache.CacheService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -48,12 +49,17 @@ class SabrInfoFetcher( } private fun logFetch(videoId: String, startTimeMs: Long, startedAt: Long, source: String): Unit { + val durationMs = System.currentTimeMillis() - startedAt logger.info( "sabr_info_fetch videoId={} startTimeMs={} source={} elapsedMs={}", videoId, startTimeMs, source, - System.currentTimeMillis() - startedAt, + durationMs, + ) + PlaybackTraceLog.record( + "sabr_info_fetch", + "videoId=$videoId startTimeMs=$startTimeMs source=$source durationMs=$durationMs", ) } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt index 8ca271a9..1aadb0f9 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PipePipeStreamService.kt @@ -1,5 +1,6 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog import dev.typetype.server.cache.CacheJson import dev.typetype.server.cache.CacheService import dev.typetype.server.models.ExtractionResult @@ -44,37 +45,62 @@ class PipePipeStreamService( withContext(Dispatchers.IO) { runCatching { withExtractionRetry { - val service = NewPipe.getServiceByUrl(url) - val linkHandler = service.streamLHFactory.fromUrl(url) - val extractor: StreamExtractor = service.getStreamExtractor(linkHandler) - withTimeout(30_000L) { runPipePipeCall { extractor.fetchPage() } } + val (service, linkHandler, extractor) = playbackTracePhase("extractor_setup") { + val service = NewPipe.getServiceByUrl(url) + val linkHandler = service.streamLHFactory.fromUrl(url) + val extractor: StreamExtractor = service.getStreamExtractor(linkHandler) + Triple(service, linkHandler, extractor) + } + playbackTracePhase("extractor_fetch_page") { + withTimeout(30_000L) { runPipePipeCall { extractor.fetchPage() } } + } coroutineScope { val streamInfoDeferred = async { - withTimeout(30_000L) { runPipePipeCall { StreamInfo.getInfo(extractor) } } + playbackTracePhase("stream_info") { + withTimeout(30_000L) { runPipePipeCall { StreamInfo.getInfo(extractor) } } + } + } + val segmentsDeferred = async { + playbackTracePhase("sponsorblock") { resolveSegments(extractor) } } - val segmentsDeferred = async { resolveSegments(extractor) } val streamInfo = streamInfoDeferred.await() rememberSabrInfo(streamInfo) streamInfo.setSponsorBlockSegments(segmentsDeferred.await()) val response = StreamAudioContractResolver.apply(streamInfo.toStreamResponse()) - val withSubtitles = if ( - fetchSupplementalSubtitles && - response.subtitles.isEmpty() && - service.serviceId == 0 - ) { - val subtitles = try { - subtitleService.fetchSubtitles(streamInfo.id) - } catch (error: CancellationException) { - throw error - } catch (_: Exception) { - emptyList() + val withSubtitles = when { + !fetchSupplementalSubtitles -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=disabled") + } + response.subtitles.isNotEmpty() -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=already_present") + } + service.serviceId != 0 -> response.also { + PlaybackTraceLog.record("supplemental_subtitles", "status=not_applicable") + } + else -> { + var subtitleStatus = "loaded" + val subtitles = playbackTracePhase("supplemental_subtitle_fetch") { + try { + subtitleService.fetchSubtitles(streamInfo.id) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + subtitleStatus = "failed:${error.javaClass.simpleName}" + emptyList() + } + } + PlaybackTraceLog.record( + "supplemental_subtitles", + "status=$subtitleStatus count=${subtitles.size}", + ) + response.copy(subtitles = subtitles) } - response.copy(subtitles = subtitles) - } else { - response } - if (service.serviceId == BILIBILI_SERVICE_ID) bilibiliRelatedService.patchRelatedStreams(withSubtitles, linkHandler.url) - else withSubtitles + if (service.serviceId == BILIBILI_SERVICE_ID) { + playbackTracePhase("bilibili_related_streams") { + bilibiliRelatedService.patchRelatedStreams(withSubtitles, linkHandler.url) + } + } else withSubtitles } } }.fold( @@ -132,5 +158,24 @@ class PipePipeStreamService( } } +private suspend fun playbackTracePhase(phase: String, block: suspend () -> T): T { + val startedAt = System.nanoTime() + var outcome = "ok" + try { + return block() + } catch (error: CancellationException) { + outcome = "cancelled" + throw error + } catch (error: Exception) { + outcome = "error:${error.javaClass.simpleName}" + throw error + } finally { + PlaybackTraceLog.record( + "pipepipe_phase", + "phase=$phase outcome=$outcome durationMs=${(System.nanoTime() - startedAt) / 1_000_000}", + ) + } +} + suspend fun runPipePipeCall(block: () -> T): T = runInterruptible(context = Dispatchers.IO, block = block) From 1596ce20d9f29ae42e69743c9a35778779188db1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 03:23:08 +0000 Subject: [PATCH 104/137] docs: document MSE playback trace events --- PERF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PERF.md b/PERF.md index f151f70c..653c9ec5 100644 --- a/PERF.md +++ b/PERF.md @@ -27,7 +27,7 @@ Start a video from its card. In the console, collect all `[typetype] playback.*` The same trace ID is sent on selected stream, comments, subtitle, SABR prewarm, and SABR MSE requests. Server echoes it and forwards it, with the request ID, to Token. Filter Server and Token runtime logs for `[playback_trace]` and the same `traceId`. -Browser events include API start/end/status/duration, video readiness and playback events, the first rendered frame, same-origin SABR resource timing and transfer sizes, and supported long-task observations. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. +Browser events include API start/end/status/duration; video readiness/playback and first-frame callbacks; same-origin SABR resource timing/transfer sizes; and supported long-task observations. MSE events include `mse_state`, `mse_manifest` (generation and segment count), `mse_quality`, `mse_segment_appended` (track, init/media phase, start and duration), `mse_buffer`, `mse_seek`, and `mse_error`. Segment URLs are omitted. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. Compare the browser's full API duration with the Server request duration and the nested Token phases. This separates browser/MSE waiting, Server work, and Token work without treating a fast response header as a rendered frame. From b25a63266b696a96b3a3e2584015d61430ce1456 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 03:50:18 +0000 Subject: [PATCH 105/137] docs: record traced browser startup samples --- PERF.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PERF.md b/PERF.md index 653c9ec5..355d1714 100644 --- a/PERF.md +++ b/PERF.md @@ -48,6 +48,15 @@ These are two prewarmed samples, not cold-start measurements or a percentile. Br A later card-click sample on video `ynKvUYpu1Qw` measured 1,892 ms to first frame in Chromium and 1,913 ms in Firefox. Playback started at 1,908/1,941 ms, continued for 3.94/4.01 seconds, and rendered 1920x1080 without a media error. Chromium's stream-info call took 1,291 ms and bootstrap took 172 ms; session creation took 318 ms for the consumer and 139 ms for prewarm. Firefox ran after Chromium had used the same video: stream-info took 43 ms, session calls 29-30 ms, and init calls 26-27 ms. No text tracks were attached at first frame. Firefox was cache-warm, and Chromium's origin-side cache state was not measured, so these results do not prove a cold-start pass. +The browser MSE trace was also captured for `GlosP5N7DkA` in fresh Chromium and Firefox contexts. A separate `stream_resolve` request was active around navigation from search results, and the click path used SABR prewarm plus session handoff; these are not cold-start measurements. + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,678 ms | 1,713 ms | 3.96 s | Pass | +| Firefox | 1,539 ms | 1,549 ms | 4.04 s | Pass | + +Both rendered a 1920x1080 frame with no media error or intercepted pause call. Firefox's click trace recorded the MSE manifest ready at 1,436 ms, `loadeddata` at 1,512 ms, and `first_frame` at 1,541 ms; its bootstrap trace duration was 284 ms and the prewarm/consumer session requests took 26/25 ms. Both traces showed two text tracks and zero visible tracks at first frame. This confirms a sub-3-second start for these prewarmed samples, not every subtitle format or a cold start. + The TypeType SABR extractor configuration disables supplemental subtitle lookup. This shows subtitles did not gate this start, not that every subtitle path is non-blocking. A separate cold direct Stack Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. From 4094231cd46878dbb49ce064b6fe6333554a5332 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 04:52:02 +0000 Subject: [PATCH 106/137] perf: trace SABR initialization preload timing --- .../SabrPlaybackInitializationPreloader.kt | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt index 89208786..f8c6d14d 100644 --- a/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt +++ b/server-playback/src/main/kotlin/dev/typetype/server/services/SabrPlaybackInitializationPreloader.kt @@ -1,5 +1,8 @@ package dev.typetype.server.services +import dev.typetype.server.PlaybackTraceLog +import dev.typetype.server.sabr.YoutubeSabrFormat +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withTimeoutOrNull @@ -28,9 +31,42 @@ object SabrPlaybackInitializationPreloader { coroutineScope { val video = holder.videoFormat .takeUnless { audioOnly } - ?.let { format -> async { sessionStore.fetchInitializationData(holder, format) } } - val audio = async { sessionStore.fetchInitializationData(holder, holder.audioFormat) } + ?.let { format -> async { fetchTrack(sessionStore, holder, format, "video") } } + val audio = async { fetchTrack(sessionStore, holder, holder.audioFormat, "audio") } SabrPlaybackInitializationPreloadResult(video?.await(), audio.await()) } } ?: SabrPlaybackInitializationPreloadResult(null, null) + + private suspend fun fetchTrack( + sessionStore: SabrSessionStore, + holder: SabrSessionHolder, + format: YoutubeSabrFormat, + track: String, + ): ByteArray? { + val startedAt = System.nanoTime() + val fields = "videoId=${holder.key.videoId} track=$track itag=${format.itag}" + PlaybackTraceLog.record("sabr_init_preload_start", fields) + val bytes = try { + sessionStore.fetchInitializationData(holder, format) + } catch (error: CancellationException) { + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=cancelled", + ) + throw error + } catch (error: Exception) { + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=failed errorType=${error.javaClass.simpleName}", + ) + throw error + } + PlaybackTraceLog.record( + "sabr_init_preload_complete", + "$fields durationMs=${elapsedMs(startedAt)} result=${if (bytes == null) "miss" else "hit"} bytes=${bytes?.size ?: 0}", + ) + return bytes + } + + private fun elapsedMs(startedAt: Long): Long = (System.nanoTime() - startedAt) / 1_000_000 } From 3869c1825f40b00faafc7deedb0f1d39e875c5d3 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 04:52:14 +0000 Subject: [PATCH 107/137] docs: record latest Beta playback timings --- PERF.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/PERF.md b/PERF.md index 355d1714..943baef6 100644 --- a/PERF.md +++ b/PERF.md @@ -27,7 +27,7 @@ Start a video from its card. In the console, collect all `[typetype] playback.*` The same trace ID is sent on selected stream, comments, subtitle, SABR prewarm, and SABR MSE requests. Server echoes it and forwards it, with the request ID, to Token. Filter Server and Token runtime logs for `[playback_trace]` and the same `traceId`. -Browser events include API start/end/status/duration; video readiness/playback and first-frame callbacks; same-origin SABR resource timing/transfer sizes; and supported long-task observations. MSE events include `mse_state`, `mse_manifest` (generation and segment count), `mse_quality`, `mse_segment_appended` (track, init/media phase, start and duration), `mse_buffer`, `mse_seek`, and `mse_error`. Segment URLs are omitted. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. +Browser events include API start/end/status/duration; video readiness/playback and first-frame callbacks; same-origin SABR resource timing/transfer sizes; and supported long-task observations. MSE events include `mse_state`, `mse_manifest` (generation and segment count), `mse_quality`, `mse_segment_appended` (track, init/media phase, start and duration), `mse_buffer`, `mse_seek`, and `mse_error`. Segment URLs are omitted. Server events include normalized HTTP route/status/duration, SABR info resolution and session preparation, and `sabr_init_preload_start`/`sabr_init_preload_complete` with per-track duration, result and byte count. Token events include session cache/singleflight state and durations for visitor data, BotGuard, GenerateIT, PO-token minting, YouTube session creation, and SABR phases. Compare the browser's full API duration with the Server request duration and the nested Token phases. This separates browser/MSE waiting, Server work, and Token work without treating a fast response header as a rendered frame. @@ -59,10 +59,23 @@ Both rendered a 1920x1080 frame with no media error or intercepted pause call. F The TypeType SABR extractor configuration disables supplemental subtitle lookup. This shows subtitles did not gate this start, not that every subtitle path is non-blocking. -A separate cold direct Stack Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. +A separate cold direct Beta stream-info request returned 200 in 3,585 ms. Its correlated Server log measured the PO-token request at 1,079 ms and the SABR Token session at 196 ms. Token logs broke the cold token refresh into visitor-data fetch 35 ms, BotGuard challenge 178 ms, BotGuard execution 806 ms, GenerateIT fetch 40 ms, and token minting under 10 ms. The later SABR session phases totaled 184 ms (Innertube 105 ms, player 59 ms, session build 19 ms). This endpoint measures extraction, not click-to-first-frame; the timings identify BotGuard as a significant measured phase but do not attribute the entire extraction delay to Token. -The direct Stack request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Stack-local Server or Token container logs, so public-runtime log correlation is not yet verified. The new trace-gated Server events split provider setup, page fetch, stream extraction, SponsorBlock, supplemental subtitles, authenticated resolution, SABR bootstrap metadata, and format preparation. Verify they appear with the browser trace ID after the change reaches Beta before attributing public latency to a backend phase. +The direct Beta request correlated the same trace and request IDs through Server and Token. Browser trace IDs from the public Beta runs did not appear in the Beta Server or Token logs, so public-runtime log correlation is not yet verified. The new trace-gated Server events split provider setup, page fetch, stream extraction, SponsorBlock, supplemental subtitles, authenticated resolution, SABR bootstrap metadata, format preparation, and per-track initialization preload. Verify they appear with the browser trace ID after the change reaches Beta before attributing public latency to a backend phase. + +## Latest Beta Spot Check (2026-09-25) + +Fresh browser contexts on `jNQXAC9IVRw` used card-click SABR prewarm and rendered 320x240 frames: + +| Browser | Click to first frame | Click to playing | Playback advanced | Result | +| --- | ---: | ---: | ---: | --- | +| Chromium | 1,464 ms | 1,464 ms | 3.93 s of 4 s | Pass | +| Firefox | 1,494 ms | 1,529 ms | 3.99 s of 4 s | Pass | + +Neither browser reported a media error. In Firefox, stream/bootstrap resolution took 181 ms, the full stream-info path 318 ms (including a concurrent 163 ms `stream_resolve`), session creation took 26 ms per request, init responses took 24-25 ms, and the first video/audio segment responses took 25/37 ms. Comments took about 941 ms and completed before the first frame, so they did not gate this sample. Two text tracks were attached, neither visible; no subtitle request was observed on the first-frame critical path. + +These were prewarmed card-click samples; fresh browser contexts do not establish a cold Server/cache state. The observed 240p output is not evidence of high-resolution startup or a general latency distribution. The earlier network logs include SABR create calls taking about 5.0-11.3 seconds, including a request selecting itag 299; without matching trace IDs and Server/Token phase records, those outliers cannot be attributed to Token or Server. The new per-track initialization events should show whether the synchronous init preloader is on the delay path once deployed and correlated. The under-3-second cold-start goal remains unverified. ## Acceptance -A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The four browser samples above pass individually. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. +A measured run passes only when click-to-`playback.first_frame.elapsedMs` is below 3000 ms on Beta. The earlier samples and latest spot checks pass individually. Keep matching browser events and Server/Token records by trace ID, with the video and whether the run was cold or prewarmed. The overall target remains unverified until cold starts and representative network conditions are measured. From dca0cb5907e9753abb6d26e490d193b9c45460b7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 06:42:39 +0000 Subject: [PATCH 108/137] perf: start SABR with a playable partial window --- .../typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt | 2 +- .../server/services/SabrPlaybackGranularRoutesTest.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt index 4a35adda..5689d902 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -143,5 +143,5 @@ internal data class TrackBuildResult( val atEnd: Boolean, ) { fun covers(requiredEndMs: Long): Boolean = - blockedBy == null && (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs + (track.segments.isNotEmpty() || atEnd) && coveredEndMs >= requiredEndMs } diff --git a/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt index 29e31f97..97f662ad 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/services/SabrPlaybackGranularRoutesTest.kt @@ -130,8 +130,8 @@ class SabrPlaybackGranularRoutesTest { setBody(windowBody()) } - assertEquals(HttpStatusCode.OK, response.status) - assertTrue(response.bodyAsText().contains("segment/1")) + assertEquals(HttpStatusCode.Accepted, response.status) + assertTrue(response.bodyAsText().contains("video:136:1 pending")) verify(atLeast = 1) { store.requestSegmentDemand(holder, any(), holder.activeGeneration()) } } From c2e3a6f5f0c8228fb149f5842baae75eb4069ca8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 06:42:39 +0000 Subject: [PATCH 109/137] test: verify token playback fallback --- .../typetype/server/services/SabrInitializationPolicyTest.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt index 20fd00e9..0270fcf6 100644 --- a/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/SabrInitializationPolicyTest.kt @@ -84,7 +84,6 @@ class SabrFallbackStreamServiceTest { val sessionStore = mockk() val tokenSessionClient = mockk() coEvery { delegate.getStreamInfo(YOUTUBE_URL) } returns ExtractionResult.Failure("MWEB player response is not valid") - coEvery { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } returns preparedInfo() coEvery { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } returns tokenSession() val service = SabrFallbackStreamService(delegate, sessionStore, tokenSessionClient) @@ -98,7 +97,7 @@ class SabrFallbackStreamServiceTest { assertEquals(3554L, response.duration) assertEquals(listOf(137), response.videoOnlyStreams.map { it.itag }) assertEquals(listOf(140), response.audioStreams.map { it.itag }) - coVerify(exactly = 1) { sessionStore.fetchInfo(VIDEO_ID, cachedFirst = true) } + coVerify(exactly = 1) { tokenSessionClient.fetchPlaybackSession(VIDEO_ID) } } @Test From 23bc6b405a3a372a41294b4ec3f9417128ca1a46 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 07:14:16 +0000 Subject: [PATCH 110/137] fix: skip arm64 Docker build on dev --- .github/workflows/docker.yml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d1c3232c..5c9f44b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,7 @@ jobs: image: ${{ steps.build-info.outputs.image }} labels: ${{ steps.meta.outputs.labels }} metadata-json: ${{ steps.meta.outputs.json }} + platform-matrix: ${{ steps.build-info.outputs.platform-matrix }} version: ${{ steps.build-info.outputs.version }} steps: @@ -54,10 +55,16 @@ jobs: else image="${REGISTRY}/${GITHUB_REPOSITORY,,}" fi + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + platform_matrix='{"include":[{"platform":"linux/amd64","arch":"amd64"}]}' + else + platform_matrix='{"include":[{"platform":"linux/amd64","arch":"amd64"},{"platform":"linux/arm64","arch":"arm64"}]}' + fi { echo "image=$image" echo "version=$version" echo "build-time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + echo "platform-matrix=$platform_matrix" } >> "$GITHUB_OUTPUT" - name: Extract metadata @@ -87,12 +94,7 @@ jobs: packages: write strategy: fail-fast: false - matrix: - include: - - platform: linux/amd64 - arch: amd64 - - platform: linux/arm64 - arch: arm64 + matrix: ${{ fromJSON(needs.prepare-image.outputs.platform-matrix) }} steps: - name: Checkout @@ -195,8 +197,12 @@ jobs: run: | cd "$DIGEST_DIR" mapfile -t digests < <(find . -maxdepth 1 -type f -printf '%f\n' | sort) - if [[ "${#digests[@]}" -ne 2 ]]; then - echo "Expected two platform digests, found ${#digests[@]}" + expected_digest_count=2 + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + expected_digest_count=1 + fi + if [[ "${#digests[@]}" -ne "$expected_digest_count" ]]; then + echo "Expected $expected_digest_count platform digest(s), found ${#digests[@]}" exit 1 fi @@ -231,8 +237,13 @@ jobs: VERSION: ${{ needs.prepare-image.outputs.version }} run: | platforms="$(docker buildx imagetools inspect "$IMAGE:$VERSION" --format '{{json .Manifest}}' | jq -r '.manifests[].platform | "\(.os)/\(.architecture)"' | sort -u)" - if [[ "$platforms" != $'linux/amd64\nlinux/arm64' ]]; then - printf 'Unexpected manifest platforms:\n%s\n' "$platforms" + expected_platforms=$'linux/amd64\nlinux/arm64' + if [[ "$GITHUB_REF_NAME" == "dev" ]]; then + expected_platforms='linux/amd64' + fi + if [[ "$platforms" != "$expected_platforms" ]]; then + printf 'Unexpected manifest platforms:\n%s\nExpected:\n%s\n' \ + "$platforms" "$expected_platforms" exit 1 fi From da342ad27143a9efcc0c160f7a22aa7ebfbf2b18 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 13:23:52 +0000 Subject: [PATCH 111/137] feat: persist video layout preferences --- openapi/components/access-control.yaml | 5 +++++ .../typetype/server/models/SettingsItem.kt | 2 ++ .../server/db/SettingsSchemaMigrations.kt | 2 ++ .../server/db/tables/SettingsTable.kt | 2 ++ .../services/SettingsPersistenceMappers.kt | 8 +++++++ .../dev/typetype/server/SettingsRoutesTest.kt | 22 +++++++++++++++++++ 6 files changed, 41 insertions(+) diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index 5e72fa9b..94554bdc 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -10,6 +10,11 @@ SettingsItem: maximum: 4 default: 1 defaultLandingPage: { type: string, default: home } + videoGridColumns: + type: integer + enum: [0, 4, 5, 6] + default: 0 + relatedVideoSize: { type: string, enum: [default, large], default: default } autoplay: { type: boolean, default: true } autoplayOnOpen: { type: boolean, default: true } autoplayCountdownSeconds: diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index 40d241f7..e75da1f9 100644 --- a/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -8,6 +8,8 @@ data class SettingsItem( val defaultQuality: String = "1080p", val defaultPlaybackSpeed: Double = 1.0, val defaultLandingPage: String = "home", + val videoGridColumns: Int = 0, + val relatedVideoSize: String = "default", val autoplay: Boolean = true, val autoplayOnOpen: Boolean = true, val autoplayCountdownSeconds: Int = 10, diff --git a/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index 2a0338c8..d8d83fb1 100644 --- a/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -34,6 +34,8 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS autoplay_countdown_seconds INTEGER NOT NULL DEFAULT 10") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS default_landing_page TEXT NOT NULL DEFAULT 'home'") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS video_grid_columns INTEGER NOT NULL DEFAULT 0") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS related_video_size TEXT NOT NULL DEFAULT 'default'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode_admin_managed BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS access_mode_admin_managed_at BIGINT NOT NULL DEFAULT 0") diff --git a/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index 8fbd1523..7a51616c 100644 --- a/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/server-db/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -8,6 +8,8 @@ object SettingsTable : Table("settings") { val defaultQuality = text("default_quality").default("1080p") val defaultPlaybackSpeed = double("default_playback_speed").default(1.0) val defaultLandingPage = text("default_landing_page").default("home") + val videoGridColumns = integer("video_grid_columns").default(0) + val relatedVideoSize = text("related_video_size").default("default") val autoplay = bool("autoplay").default(true) val autoplayOnOpen = bool("autoplay_on_open").default(true) val autoplayCountdownSeconds = integer("autoplay_countdown_seconds").default(10) diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index e119b172..868eaa64 100644 --- a/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -21,6 +21,8 @@ fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( defaultQuality = this[SettingsTable.defaultQuality], defaultPlaybackSpeed = this[SettingsTable.defaultPlaybackSpeed], defaultLandingPage = this[SettingsTable.defaultLandingPage], + videoGridColumns = this[SettingsTable.videoGridColumns], + relatedVideoSize = this[SettingsTable.relatedVideoSize], autoplay = this[SettingsTable.autoplay], autoplayOnOpen = this[SettingsTable.autoplayOnOpen], autoplayCountdownSeconds = this[SettingsTable.autoplayCountdownSeconds], @@ -63,6 +65,8 @@ fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.defaultQuality] = settings.defaultQuality this[SettingsTable.defaultPlaybackSpeed] = settings.defaultPlaybackSpeed this[SettingsTable.defaultLandingPage] = settings.defaultLandingPage + this[SettingsTable.videoGridColumns] = settings.videoGridColumns + this[SettingsTable.relatedVideoSize] = settings.relatedVideoSize this[SettingsTable.autoplay] = settings.autoplay this[SettingsTable.autoplayOnOpen] = settings.autoplayOnOpen this[SettingsTable.autoplayCountdownSeconds] = settings.autoplayCountdownSeconds @@ -104,6 +108,8 @@ fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { fun SettingsItem.normalized(): SettingsItem = copy( defaultLandingPage = defaultLandingPage.ifBlank { "home" }, + videoGridColumns = videoGridColumns.takeIf { it in VIDEO_GRID_COLUMNS } ?: 0, + relatedVideoSize = relatedVideoSize.takeIf { it in RELATED_VIDEO_SIZES } ?: "default", defaultPlaybackSpeed = defaultPlaybackSpeed.takeIf { it.isFinite() } ?.coerceIn(MIN_PLAYBACK_SPEED, MAX_PLAYBACK_SPEED) ?: 1.0, accessMode = accessMode.toAccessMode(), @@ -116,6 +122,8 @@ fun SettingsItem.normalized(): SettingsItem = copy( ) private val DEARROW_TITLE_MODES = setOf("original", "dearrow") +private val VIDEO_GRID_COLUMNS = setOf(0, 4, 5, 6) +private val RELATED_VIDEO_SIZES = setOf("default", "large") private const val MIN_PLAYBACK_SPEED = 0.25 private const val MAX_PLAYBACK_SPEED = 4.0 private val DEARROW_THUMBNAIL_MODES = setOf("original", "dearrow", "random", "dearrow_or_random") diff --git a/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt index f2fc67f3..7c22fc8c 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/SettingsRoutesTest.kt @@ -62,6 +62,8 @@ class SettingsRoutesTest { assertTrue(body.contains("\"notificationPopupsEnabled\":true")) assertTrue(body.contains("\"defaultLandingPage\":\"home\"")) assertTrue(body.contains("\"defaultPlaybackSpeed\":1.0")) + assertTrue(body.contains("\"videoGridColumns\":0")) + assertTrue(body.contains("\"relatedVideoSize\":\"default\"")) } @Test @@ -96,9 +98,29 @@ class SettingsRoutesTest { fun `GET settings returns defaults for new fields when no row exists`() = withApp { val body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() assertContainsAll(body, listOf("\"subtitlesEnabled\":false", "\"defaultSubtitleLanguage\":\"\"", "\"defaultAudioLanguage\":\"\"", "\"preferOriginalLanguage\":false", "\"enableHighQualityPlayback\":false")) + assertContainsAll(body, listOf("\"videoGridColumns\":0", "\"relatedVideoSize\":\"default\"")) assertContainsNone(body, listOf("recommendationPersonalizationEnabled", "subscriptionSyncInterval")) } + @Test + fun `PUT settings persists and bounds layout preferences`() = withApp { + client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoGridColumns":5,"relatedVideoSize":"large"}""") + } + var body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() + assertContainsAll(body, listOf("\"videoGridColumns\":5", "\"relatedVideoSize\":\"large\"")) + + client.put("/settings") { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + headers.append(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + setBody("""{"videoGridColumns":9,"relatedVideoSize":"huge"}""") + } + body = client.get("/settings") { headers.append(HttpHeaders.Authorization, "Bearer test-jwt") }.bodyAsText() + assertContainsAll(body, listOf("\"videoGridColumns\":0", "\"relatedVideoSize\":\"default\"")) + } + @Test fun `PUT settings persists new fields and GET returns them`() = withApp { client.put("/settings") { From efdbbda70b77a50663c1b5e0f17f1bbf2fa59dce Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 13:23:56 +0000 Subject: [PATCH 112/137] chore: restore automatic dev validation --- .github/workflows/ci.yml | 10 ++++------ .github/workflows/coverage.yml | 6 ++---- .github/workflows/openapi.yml | 10 ++++------ 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15cfdc27..ff9408df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,10 @@ name: CI on: - # Temporarily pause automatic CI while SABR startup is profiled. - # push: - # branches: ["dev"] - # pull_request: - # branches: ["dev", "main"] - workflow_dispatch: + push: + branches: ["dev"] + pull_request: + branches: ["dev", "main"] env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e997ba26..c7f11e77 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,10 +1,8 @@ name: Coverage on: - # Temporarily pause automatic test and coverage runs. - # pull_request: - # branches: ["dev", "main"] - workflow_dispatch: + pull_request: + branches: ["dev", "main"] env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml index a44883f7..1369962f 100644 --- a/.github/workflows/openapi.yml +++ b/.github/workflows/openapi.yml @@ -1,12 +1,10 @@ name: OpenAPI on: - # Temporarily pause automatic validation during SABR profiling. - # push: - # branches: ["dev"] - # pull_request: - # branches: ["dev", "main"] - workflow_dispatch: + push: + branches: ["dev"] + pull_request: + branches: ["dev", "main"] env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true From 2ba474ea9834cc88113605ba7e396a500bdd7045 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 18:05:20 +0200 Subject: [PATCH 113/137] fix: end progressive SABR windows at stream end --- .../routes/SabrPlaybackWindowTrackBuilder.kt | 4 ++ .../SabrProgressivePlaybackWindowTest.kt | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt index 5689d902..dd297599 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -68,6 +68,10 @@ internal class SabrPlaybackWindowTrackBuilder(private val sabrSessionStore: Sabr } seq = progressive.nextSequence coveredEndMs = progressive.coveredEndMs + if (endSequence > 0 && seq > endSequence) { + atEnd = true + break + } if (coveredEndMs >= goalEndMs) break continue } diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index 43989f9d..b6463826 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -121,6 +121,51 @@ class SabrProgressivePlaybackWindowTest { assertEquals(4, result.response.audio.segments.size) } + @Test + fun `progressive window marks the final segment as end of stream`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val state = mockk(relaxed = true) + every { state.getSegmentNumberAtOrAfterTimeMs(any(), any()) } returns 1 + every { state.getEndSegment(audio) } returns 2L + every { state.getEndSegment(video) } returns 2L + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + every { session.getReadableSegment(any()) } answers { + val request = firstArg() + val durationMs = when { + request.sequenceNumber == 1 -> 4_000L + else -> 6_000L + } + readableSegment( + request.format, + sequence = request.sequenceNumber, + startMs = if (request.sequenceNumber == 1) 0L else 4_000L, + durationMs = durationMs, + ) + } + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest( + generation = 0L, + playerTimeMs = 0L, + videoItag = video.itag, + audioItag = audio.itag, + bufferGoalMs = 9_000L, + ), + ) + + assertTrue(result.isReady) + assertTrue(result.response.endOfStream) + assertEquals(2, result.response.audio.segments.size) + assertEquals(2, requireNotNull(result.response.video).segments.size) + } + private fun readableSegment( format: YoutubeSabrFormat, sequence: Int = 1, From 9843d78b64fe86b6017573f0e7c2a4bbd233a9de Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 18:16:46 +0200 Subject: [PATCH 114/137] fix: close SABR windows at indexed media end --- .../routes/SabrPlaybackWindowTrackBuilder.kt | 13 +++++-- .../SabrProgressivePlaybackWindowTest.kt | 39 +++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt index dd297599..74b2eaab 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTrackBuilder.kt @@ -32,11 +32,16 @@ internal class SabrPlaybackWindowTrackBuilder(private val sabrSessionStore: Sabr val segments = mutableListOf() var blockedBy: String? = null var blockedRequest: SabrSegmentRequest? = null - var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) - var coveredEndMs = targetMs val endSequence = if (activeLive) 0 else holder.session.streamState.getEndSegment(format).toInt() - var atEnd = false - while (segments.size < MAX_SEGMENTS_PER_TRACK) { + val endTimeMs = if (endSequence > 0) { + holder.session.streamState.getSegmentEndMs(format, endSequence).takeIf { it > 0L } ?: 0L + } else { + 0L + } + var seq = holder.playbackContinuationSequence(format, targetMs, activeLive) + var coveredEndMs = if (endTimeMs > 0L && targetMs >= endTimeMs) endTimeMs else targetMs + var atEnd = endTimeMs > 0L && targetMs >= endTimeMs + while (!atEnd && segments.size < MAX_SEGMENTS_PER_TRACK) { if (endSequence > 0 && seq > endSequence) { atEnd = true break diff --git a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt index b6463826..cbbbf6c9 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/routes/SabrProgressivePlaybackWindowTest.kt @@ -166,6 +166,45 @@ class SabrProgressivePlaybackWindowTest { assertEquals(2, requireNotNull(result.response.video).segments.size) } + @Test + fun `window at the indexed end closes without requesting past the timeline`() = runTest { + val audio = format(140, isAudio = true) + val video = format(299, isAudio = false) + val state = mockk(relaxed = true) + every { state.getEndSegment(audio) } returns 2L + every { state.getEndSegment(video) } returns 2L + every { state.getSegmentEndMs(audio, 2) } returns 20_000L + every { state.getSegmentEndMs(video, 2) } returns 10_000L + val session = mockk(relaxed = true) + every { session.streamState } returns state + every { session.getCachedSegment(any()) } returns null + val holder = holder(session, audio, video) + val store = mockk() + coEvery { store.cachedSegment(holder, any()) } returns null + val ranges = listOf( + SabrPlaybackBufferedRange(audio.itag, 0L, 20_000L), + SabrPlaybackBufferedRange(video.itag, 0L, 10_000L), + ) + + val result = SabrPlaybackWindowBuilder(store).build( + holder, + SabrPlaybackWindowRequest( + generation = 0L, + playerTimeMs = 9_000L, + videoItag = video.itag, + audioItag = audio.itag, + bufferGoalMs = 10_000L, + bufferedRanges = ranges, + ), + ) + + assertTrue(result.isReady) + assertTrue(result.response.endOfStream) + assertTrue(result.blockedRequests.isEmpty()) + assertTrue(result.response.audio.segments.isEmpty()) + assertTrue(requireNotNull(result.response.video).segments.isEmpty()) + } + private fun readableSegment( format: YoutubeSabrFormat, sequence: Int = 1, From 858cda8e8d35ce75101f2b52dbd22fa5cf1bc8e5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 19:38:22 +0200 Subject: [PATCH 115/137] fix: invalidate YouTube session on SABR timeout --- .../YoutubeSessionSabrStreamService.kt | 12 +++++---- .../services/YoutubeSessionStreamService.kt | 4 +++ .../services/AuthenticatedSabrTimeoutTest.kt | 25 +++++++++++++++++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt index cbb67120..b93c2667 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionSabrStreamService.kt @@ -26,17 +26,19 @@ class YoutubeSessionSabrStreamService( AuthenticatedSabrInfoResult.Failed -> ExtractionResult.Failure("Authenticated SABR playback unavailable") AuthenticatedSabrInfoResult.TimedOut -> - ExtractionResult.Failure( - "Authenticated SABR preparation timed out", - AuthenticatedSabrPolicy.TIMEOUT_CODE, - ) + reconnectResult(userId) AuthenticatedSabrInfoResult.NotConnected -> null } } } catch (error: TimeoutCancellationException) { - ExtractionResult.Failure("Authenticated SABR preparation timed out", AuthenticatedSabrPolicy.TIMEOUT_CODE) + reconnectResult(userId) } } + + private suspend fun reconnectResult(userId: String): ExtractionResult { + metadataService.markYoutubeSessionNeedsReconnect(userId) + return ExtractionResult.BadRequest(YOUTUBE_SESSION_RECONNECT_ERROR, YOUTUBE_SESSION_RECONNECT_CODE) + } } fun ExtractionResult.requiresYoutubeSession(): Boolean = diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt index a0759f72..214d68bb 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/YoutubeSessionStreamService.kt @@ -45,6 +45,10 @@ class YoutubeSessionStreamService( return result } + suspend fun markYoutubeSessionNeedsReconnect(userId: String) { + youtubeSessionService.markNeedsReconnect(userId) + } + private fun StreamResponse.withSignedHlsUrl( credentials: YoutubeSessionCredentials, url: String, diff --git a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt index 32501d89..c0e3840d 100644 --- a/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/AuthenticatedSabrTimeoutTest.kt @@ -1,7 +1,9 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.StreamResponse import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.delay @@ -70,11 +72,30 @@ class AuthenticatedSabrTimeoutTest { mockk(relaxed = true), timeoutMs = 20L, ) + coEvery { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } returns Unit val result = withTimeout(1_000L) { service.getStreamInfo(USER_ID, URL) } - assertTrue(result is ExtractionResult.Failure) - assertEquals(AuthenticatedSabrPolicy.TIMEOUT_CODE, (result as ExtractionResult.Failure).code) + assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) + coVerify { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } + } + + @Test + fun `authenticated SABR probe timeout marks session needs reconnect`() = runTest { + val metadata = mockk() + val info = mockk() + val stream = mockk(relaxed = true) + coEvery { metadata.getStreamInfo(USER_ID, URL) } returns ExtractionResult.Success(stream) + coEvery { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } returns Unit + coEvery { info.fetch(USER_ID, VIDEO_ID) } returns AuthenticatedSabrInfoResult.TimedOut + val service = YoutubeSessionSabrStreamService(metadata, info) + + val result = service.getStreamInfo(USER_ID, URL) + + assertTrue(result is ExtractionResult.BadRequest) + assertEquals(YOUTUBE_SESSION_RECONNECT_CODE, (result as ExtractionResult.BadRequest).code) + coVerify { metadata.markYoutubeSessionNeedsReconnect(USER_ID) } } private fun service( From 51b9d2daa2e32ed411181319f6c2237fdfeb5aa1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 21:41:04 +0200 Subject: [PATCH 116/137] fix: continue metadata repair across batches --- .../UserVideoMetadataRepairService.kt | 67 ++++++++++++++----- .../server/services/VideoMetadataResolver.kt | 47 ++++++++++--- .../UserVideoMetadataRepairRoutesTest.kt | 67 ++++++++++++++++++- 3 files changed, 153 insertions(+), 28 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt index 2f1757ec..9d50e6d4 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt @@ -7,6 +7,7 @@ import dev.typetype.server.db.tables.WatchLaterTable import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.launch import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -24,22 +25,35 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private val lastScheduledAt = ConcurrentHashMap() fun schedulePlaylists(scope: CoroutineScope, userId: String): Unit = schedule(scope, "playlists:$userId") { - repairPlaylists(userId) + drain { excluded -> repairPlaylists(userId, excluded) } } fun scheduleWatchLater(scope: CoroutineScope, userId: String): Unit = schedule(scope, "watch-later:$userId") { - repairWatchLater(userId) + drain { excluded -> repairWatchLater(userId, excluded) } } fun scheduleFavorites(scope: CoroutineScope, userId: String): Unit = schedule(scope, "favorites:$userId") { - repairFavorites(userId) + drain { excluded -> repairFavorites(userId, excluded) } } - suspend fun repairPlaylists(userId: String): Int = repair(userId, ::playlistCandidateUrls) + private suspend fun repairPlaylists(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::playlistCandidateUrls) - suspend fun repairWatchLater(userId: String): Int = repair(userId, ::watchLaterCandidateUrls) + private suspend fun repairWatchLater(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::watchLaterCandidateUrls) - suspend fun repairFavorites(userId: String): Int = repair(userId, ::favoriteCandidateUrls) + private suspend fun repairFavorites(userId: String, excluded: Set): RepairOutcome = + repair(userId, excluded, ::favoriteCandidateUrls) + + private suspend fun drain(repair: suspend (Set) -> RepairOutcome) { + val excluded = mutableSetOf() + repeat(MAX_BATCHES_PER_RUN) { + val outcome = repair(excluded) + if (outcome.attempted == 0 || outcome.updated == 0) return + excluded += outcome.attemptedUrls + delay(BATCH_DELAY_MS) + } + } private fun schedule(scope: CoroutineScope, key: String, repair: suspend () -> Unit) { val now = System.currentTimeMillis() @@ -60,22 +74,33 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver } } - private suspend fun repair(userId: String, candidates: suspend (String) -> List): Int { - val urls = candidates(userId).take(MAX_REPAIR_PER_REQUEST) - if (urls.isEmpty()) return 0 - val metadata = resolver.resolve(urls) - if (metadata.isEmpty()) return 0 - return DatabaseFactory.query { - metadata.values.sumOf { item -> + private suspend fun repair( + userId: String, + excluded: Set, + candidates: suspend (String) -> List, + ): RepairOutcome { + val urls = candidates(userId).filterNot { it in excluded }.take(MAX_REPAIR_PER_REQUEST) + if (urls.isEmpty()) return RepairOutcome(attemptedUrls = emptySet(), updated = 0) + val first = resolver.resolveDetailed(urls) + var updated = applyMetadata(userId, first) + if (first.retryableUrls.isNotEmpty()) { + delay(RETRY_DELAY_MS) + val retried = resolver.resolveDetailed(first.retryableUrls) + updated += applyMetadata(userId, retried) + } + return RepairOutcome(attemptedUrls = urls.toSet(), updated = updated) + } + + private suspend fun applyMetadata(userId: String, resolution: VideoMetadataResolution): Int = + DatabaseFactory.query { + resolution.metadata.values.sumOf { item -> updatePlaylistVideos(userId, item) + updateWatchLater(userId, item) + updateFavorites(userId, item) } } - } private suspend fun playlistCandidateUrls(userId: String): List = DatabaseFactory.query { PlaylistVideosTable.selectAll() .where { (PlaylistVideosTable.userId eq userId) and playlistNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[PlaylistVideosTable.url] } .distinct() } @@ -83,7 +108,6 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private suspend fun watchLaterCandidateUrls(userId: String): List = DatabaseFactory.query { WatchLaterTable.selectAll() .where { (WatchLaterTable.userId eq userId) and watchLaterNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[WatchLaterTable.url] } .distinct() } @@ -91,7 +115,6 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver private suspend fun favoriteCandidateUrls(userId: String): List = DatabaseFactory.query { FavoritesTable.selectAll() .where { (FavoritesTable.userId eq userId) and favoriteNeedsRepair() } - .limit(MAX_REPAIR_PER_REQUEST) .map { it[FavoritesTable.videoUrl] } .distinct() } @@ -145,6 +168,16 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver const val FALLBACK_TITLE_PATTERN = "YouTube video %" const val YOUTUBE_THUMB_PATTERN = "https://i.ytimg.com/vi/%" const val MAX_REPAIR_PER_REQUEST = 25 + const val MAX_BATCHES_PER_RUN = 8 + const val BATCH_DELAY_MS = 750L + const val RETRY_DELAY_MS = 2_000L const val REPAIR_COOLDOWN_MS = 5 * 60 * 1000L } } + +private data class RepairOutcome( + val attemptedUrls: Set, + val updated: Int, +) { + val attempted: Int get() = attemptedUrls.size +} diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt index 7a604287..b98eabb2 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/VideoMetadataResolver.kt @@ -10,6 +10,11 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +internal data class VideoMetadataResolution( + val metadata: Map, + val retryableUrls: Set, +) + class VideoMetadataResolver(private val streamService: StreamService) { suspend fun enrichPlaylistVideos(videos: List): List { val metadata = resolve(videos.filter(::shouldEnrich).map { it.url }) @@ -27,23 +32,37 @@ class VideoMetadataResolver(private val streamService: StreamService) { return distinct.map { item -> metadata[item.videoUrl]?.toFavorite(item) ?: item.withYoutubeFallbackTitle() } } - suspend fun resolve(urls: Collection): Map = coroutineScope { + suspend fun resolve(urls: Collection): Map = + resolveDetailed(urls).metadata + + internal suspend fun resolveDetailed(urls: Collection): VideoMetadataResolution = coroutineScope { val semaphore = Semaphore(MAX_CONCURRENT_RESOLUTIONS) - urls.map { it.trim() }.filter { it.isNotBlank() }.distinct().map { url -> + val results = urls.map { it.trim() }.filter { it.isNotBlank() }.distinct().map { url -> async { semaphore.withPermit { resolve(url) } } - }.awaitAll().filterNotNull().toMap() + }.awaitAll().filterNotNull() + val metadata = mutableMapOf() + val retryableUrls = mutableSetOf() + results.forEach { result -> + when (result) { + is ResolutionResult.Resolved -> metadata[result.url] = result.metadata + is ResolutionResult.Failed -> if (result.retryable) { + retryableUrls += result.url + } + } + } + VideoMetadataResolution(metadata, retryableUrls) } - private suspend fun resolve(url: String): Pair? = try { + private suspend fun resolve(url: String): ResolutionResult? = try { when (val result = streamService.getStreamInfo(url)) { - is ExtractionResult.Success -> url to result.data.toMetadata(url) - is ExtractionResult.BadRequest -> null - is ExtractionResult.Failure -> null + is ExtractionResult.Success -> ResolutionResult.Resolved(url, result.data.toMetadata(url)) + is ExtractionResult.BadRequest -> ResolutionResult.Failed(url, retryable = false) + is ExtractionResult.Failure -> ResolutionResult.Failed(url, retryable = true) } } catch (e: CancellationException) { throw e } catch (_: Exception) { - null + ResolutionResult.Failed(url, retryable = true) } private fun shouldEnrich(video: PlaylistVideoItem): Boolean = @@ -92,3 +111,15 @@ class VideoMetadataResolver(private val streamService: StreamService) { const val YOUTUBE_THUMB_PREFIX = "https://i.ytimg.com/vi/" } } + +private sealed interface ResolutionResult { + data class Resolved( + val url: String, + val metadata: VideoMetadataItem, + ) : ResolutionResult + + data class Failed( + val url: String, + val retryable: Boolean, + ) : ResolutionResult +} diff --git a/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt b/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt index 2af7e2b3..b1921819 100644 --- a/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/UserVideoMetadataRepairRoutesTest.kt @@ -138,7 +138,61 @@ class UserVideoMetadataRepairRoutesTest { repair.schedulePlaylists(scope, TEST_USER_ID) delay(100) - assertEquals(1, attempts.get()) + assertEquals(2, attempts.get()) + } finally { + scope.cancel() + } + } + + @Test + fun `repair continues across more than one metadata batch`() = runBlocking { + val attempts = AtomicInteger() + val repair = UserVideoMetadataRepairService(VideoMetadataResolver(alwaysSuccessfulStreamService(attempts))) + val playlist = playlists.create(TEST_USER_ID, PlaylistItem(name = "Imported", description = "")) + repeat(26) { index -> + val url = "https://www.youtube.com/watch?v=video$index" + playlists.addVideo(TEST_USER_ID, playlist.id, fallbackVideo(url)) + } + val scope = CoroutineScope(SupervisorJob()) + + try { + repair.schedulePlaylists(scope, TEST_USER_ID) + withTimeout(10_000) { + while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.count { it.title.startsWith("Resolved") } != 26) { + delay(20) + } + } + assertEquals(26, attempts.get()) + } finally { + scope.cancel() + } + } + + @Test + fun `transient metadata failure is retried once`() = runBlocking { + val attempts = AtomicInteger() + val streamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + return if (attempts.incrementAndGet() == 1) { + ExtractionResult.Failure("Provider blocked", "provider_access_blocked") + } else { + ExtractionResult.Success(stream(url)) + } + } + } + val repair = UserVideoMetadataRepairService(VideoMetadataResolver(streamService)) + val playlist = playlists.create(TEST_USER_ID, PlaylistItem(name = "Imported", description = "")) + playlists.addVideo(TEST_USER_ID, playlist.id, fallbackVideo(VIDEO_URL)) + val scope = CoroutineScope(SupervisorJob()) + + try { + repair.schedulePlaylists(scope, TEST_USER_ID) + withTimeout(10_000) { + while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.single()?.title != "Resolved abc123") { + delay(20) + } + } + assertEquals(2, attempts.get()) } finally { scope.cancel() } @@ -146,11 +200,18 @@ class UserVideoMetadataRepairRoutesTest { private fun fallbackVideo(url: String): PlaylistVideoItem = PlaylistVideoItem( url = url, - title = "YouTube video abc123", - thumbnail = "https://i.ytimg.com/vi/abc123/hqdefault.jpg", + title = "YouTube video ${url.substringAfterLast('=')}", + thumbnail = "https://i.ytimg.com/vi/${url.substringAfterLast('=')}/hqdefault.jpg", duration = 0L, ) + private fun alwaysSuccessfulStreamService(attempts: AtomicInteger): StreamService = object : StreamService { + override suspend fun getStreamInfo(url: String): ExtractionResult { + attempts.incrementAndGet() + return ExtractionResult.Success(stream(url)) + } + } + private fun fakeStreamService(started: CompletableDeferred, release: CompletableDeferred): StreamService = object : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult { started.complete(Unit) From 335313fa163558033159f13252046469830d9b83 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 25 Sep 2026 21:55:36 +0200 Subject: [PATCH 117/137] fix: keep metadata repair batches moving --- .../typetype/server/services/UserVideoMetadataRepairService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt index 9d50e6d4..98497d02 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UserVideoMetadataRepairService.kt @@ -49,7 +49,7 @@ class UserVideoMetadataRepairService(private val resolver: VideoMetadataResolver val excluded = mutableSetOf() repeat(MAX_BATCHES_PER_RUN) { val outcome = repair(excluded) - if (outcome.attempted == 0 || outcome.updated == 0) return + if (outcome.attempted == 0) return excluded += outcome.attemptedUrls delay(BATCH_DELAY_MS) } From 12b7bcf3b3f585a197d57bc1ee318606a00ef2dc Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 09:45:53 +0000 Subject: [PATCH 118/137] feat: add progressive YouTube Takeout imports --- openapi/components/portability.yaml | 5 + .../server/portability/PortabilityAdapter.kt | 19 ++ .../PortabilityBatchRecordSource.kt | 18 ++ .../server/portability/PortabilityEngine.kt | 8 +- .../PortabilityImportCoordinator.kt | 41 ++++ .../server/portability/PortabilityJob.kt | 9 + .../portability/PortabilityJobModels.kt | 19 ++ .../server/portability/PortabilityProgress.kt | 35 +++- .../server/portability/PortabilitySpool.kt | 22 +++ .../TypeTypePortabilityCoreImport.kt | 1 - .../YoutubeTakeoutImportWorkflow.kt | 185 ++++++++++++++++++ .../TypeTypePortabilityCoreImportTest.kt | 58 ++++++ .../YoutubeTakeoutImportWorkflowTest.kt | 141 +++++++++++++ 13 files changed, 556 insertions(+), 5 deletions(-) create mode 100644 server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt create mode 100644 server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt create mode 100644 server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt create mode 100644 server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt create mode 100644 server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt diff --git a/openapi/components/portability.yaml b/openapi/components/portability.yaml index 05302336..04351601 100644 --- a/openapi/components/portability.yaml +++ b/openapi/components/portability.yaml @@ -87,6 +87,11 @@ PortabilityJobProgress: unit: { type: string, enum: [records, categories, bytes] } processed: { type: integer, format: int64, minimum: 0 } total: { type: integer, format: int64, minimum: 0, nullable: true } + category: { allOf: [{ $ref: '#/PortabilityCategory' }], nullable: true } + stage: { type: string, enum: [subscriptions, recentHistory, playlists, history, remaining], nullable: true } + stageProcessed: { type: integer, format: int64, minimum: 0 } + stageTotal: { type: integer, format: int64, minimum: 0, nullable: true } + checkpoint: { type: integer, format: int64, minimum: 0 } PortabilityJobReport: type: object required: [id, state] diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt index 10dec38b..ad3236a1 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityAdapter.kt @@ -32,6 +32,20 @@ interface PortabilityRecordSource { fun categories(): Set fun counts(): Map fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) + fun readBatch(category: PortabilityCategory, cursor: Long?, limit: Int): PortabilityRecordBatch { + require(limit > 0) + var position = 0L + var nextCursor = cursor + val records = ArrayList(limit) + forEach(category) { record -> + if (position > (cursor ?: -1L) && records.size < limit) { + records += record + nextCursor = position + } + position++ + } + return PortabilityRecordBatch(records, nextCursor) + } fun forEachChild(category: PortabilityCategory, parentKey: String, block: (PortabilityRecord) -> Unit) { forEach(category) { record -> if (record.parentKey() == parentKey.trim().lowercase()) block(record) @@ -39,6 +53,11 @@ interface PortabilityRecordSource { } } +data class PortabilityRecordBatch( + val records: List, + val nextCursor: Long?, +) + interface PortabilityAdapter { val descriptor: PortabilityAdapterDescriptor val autoDetect: Boolean get() = true diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt new file mode 100644 index 00000000..2b37752a --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityBatchRecordSource.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.portability + +internal class PortabilityBatchRecordSource( + private val category: PortabilityCategory, + private val records: List, +) : PortabilityRecordSource { + init { + require(records.all { it.category == category }) + } + + override fun categories(): Set = setOf(category) + + override fun counts(): Map = mapOf(category to records.size.toLong()) + + override fun forEach(category: PortabilityCategory, block: (PortabilityRecord) -> Unit) { + if (category == this.category) records.forEach(block) + } +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt index 790caec8..c0fbafb8 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityEngine.kt @@ -134,12 +134,14 @@ class PortabilityEngine constructor( total, interval = portabilityProgressInterval(total), ) - val result = dataPort.import( + val result = PortabilityImportCoordinator(dataPort).apply( job.ownerId, source, request, - onCategoryProgress = { _, count -> progress.add(count) }, - onCategoryComplete = { _, _ -> }, + job.snapshot().preview?.detection?.format, + counts, + progress, + job::updateResult, ) progress.finish() job.transition(setOf(PortabilityJobState.APPLYING), PortabilityJobState.COMPLETED, result = result) diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt new file mode 100644 index 00000000..363dfa2e --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityImportCoordinator.kt @@ -0,0 +1,41 @@ +package dev.typetype.server.portability + +internal class PortabilityImportCoordinator( + private val dataPort: PortabilityDataPort, +) { + suspend fun apply( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + format: PortabilityFormat?, + counts: Map, + progress: PortabilityProgressReporter, + onCommitted: (Map) -> Unit, + ): Map { + if (format == PortabilityFormat.YOUTUBE_TAKEOUT) { + return YoutubeTakeoutImportWorkflow(dataPort).apply( + userId, + source, + request, + progress, + onCommitted, + ) + } + val completed = linkedMapOf() + return dataPort.import( + userId, + source, + request, + onCategoryProgress = { category, count -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + progress.add(count) + }, + onCategoryComplete = { category, count -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + completed[category.wireName] = count + onCommitted(completed.toMap()) + progress.checkpoint() + }, + ) + } +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt index cd978274..c0ca4127 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJob.kt @@ -46,6 +46,15 @@ class PortabilityJob( } } + fun updateResult(result: Map) { + while (true) { + val current = value.get() + if (current.state in TERMINAL_STATES) return + val next = current.copy(updatedAt = clock(), result = result) + if (value.compareAndSet(current, next)) return + } + } + fun transition( expected: Set, state: PortabilityJobState, diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt index 97fffcfa..726ed8ef 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityJobModels.kt @@ -53,12 +53,31 @@ enum class PortabilityProgressUnit { BYTES, } +@Serializable +enum class PortabilityImportStage { + @SerialName("subscriptions") + SUBSCRIPTIONS, + @SerialName("recentHistory") + RECENT_HISTORY, + @SerialName("playlists") + PLAYLISTS, + @SerialName("history") + HISTORY, + @SerialName("remaining") + REMAINING, +} + @Serializable data class PortabilityJobProgress( val phase: PortabilityProgressPhase, val unit: PortabilityProgressUnit, val processed: Long, val total: Long? = null, + val category: PortabilityCategory? = null, + val stage: PortabilityImportStage? = null, + val stageProcessed: Long = 0L, + val stageTotal: Long? = null, + val checkpoint: Long = 0L, ) @Serializable diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt index bc8c88eb..55abca6c 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityProgress.kt @@ -13,18 +13,39 @@ class PortabilityProgressReporter( ) { private var processed = 0L private var published = -1L + private var category: PortabilityCategory? = null + private var stage: PortabilityImportStage? = null + private var stageProcessed = 0L + private var stageTotal: Long? = null + private var checkpoint = 0L init { publish(force = true) } + fun setStage(stage: PortabilityImportStage, category: PortabilityCategory, total: Long?) { + if (this.stage == stage && this.category == category && stageTotal == total) return + this.stage = stage + this.category = category + stageProcessed = 0L + stageTotal = total + publish(force = true) + } + fun add(count: Long = 1L) { ensureActive() require(count >= 0L) processed = Math.addExact(processed, count) + stageProcessed = Math.addExact(stageProcessed, count) publish(force = false) } + fun checkpoint() { + ensureActive() + checkpoint = Math.addExact(checkpoint, 1L) + publish(force = true) + } + fun finish() { ensureActive() publish(force = true) @@ -36,7 +57,19 @@ class PortabilityProgressReporter( private fun publish(force: Boolean) { if (!force && processed - published < interval) return - job.updateProgress(PortabilityJobProgress(phase, unit, processed, total)) + job.updateProgress( + PortabilityJobProgress( + phase, + unit, + processed, + total, + category, + stage, + stageProcessed, + stageTotal, + checkpoint, + ), + ) published = processed } } diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt index 24acc1d5..a6261358 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilitySpool.kt @@ -157,6 +157,28 @@ class PortabilitySpool private constructor( } } + override fun readBatch(category: PortabilityCategory, cursor: Long?, limit: Int): PortabilityRecordBatch { + require(limit > 0) + flush() + connection.prepareStatement( + "SELECT ordinal, payload FROM records WHERE category = ? AND ordinal > ? ORDER BY ordinal LIMIT ?", + ).use { statement -> + statement.setString(1, category.wireName) + statement.setLong(2, cursor ?: 0L) + statement.setInt(3, limit) + statement.executeQuery().use { rows -> + var nextCursor = cursor + val records = buildList { + while (rows.next()) { + nextCursor = rows.getLong(1) + add(CacheJson.decodeFromString(rows.getString(2))) + } + } + return PortabilityRecordBatch(records, nextCursor) + } + } + } + override fun forEachChild( category: PortabilityCategory, parentKey: String, diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt index c93c5d11..d0c99e05 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImport.kt @@ -124,7 +124,6 @@ internal object TypeTypePortabilityCoreImport { var count = 0L source.forEach(PortabilityCategory.PLAYLISTS) { record -> if (record is PortabilityPlaylist) count += insertPlaylist(userId, record) - onRecord() } source.forEach(PortabilityCategory.PLAYLISTS) { record -> if (record is PortabilityPlaylistVideo) count += insertPlaylistVideo(userId, record) diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt new file mode 100644 index 00000000..cb54c335 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflow.kt @@ -0,0 +1,185 @@ +package dev.typetype.server.portability + +import java.util.PriorityQueue + +internal class YoutubeTakeoutImportWorkflow( + private val dataPort: PortabilityDataPort, +) { + suspend fun apply( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + progress: PortabilityProgressReporter, + onCommitted: (Map) -> Unit, + ): Map { + val counts = source.counts() + val selected = request.categories + val result = linkedMapOf() + val started = mutableSetOf() + val recentHistoryKeys = hashSetOf() + var remainingHistory = 0L + + suspend fun commit( + category: PortabilityCategory, + records: List, + stage: PortabilityImportStage, + stageTotal: Long?, + ) { + progress.setStage(stage, category, stageTotal) + val policy = if (started.add(category)) request.duplicatePolicy else PortabilityDuplicatePolicy.SKIP + val imported = dataPort.import( + userId, + PortabilityBatchRecordSource(category, records), + PortabilityImportRequest(setOf(category), policy), + onCategoryProgress = { _, count -> progress.add(count) }, + )[category.wireName] ?: 0L + result[category.wireName] = Math.addExact(result[category.wireName] ?: 0L, imported) + onCommitted(result.toMap()) + progress.checkpoint() + } + + if (PortabilityCategory.SUBSCRIPTIONS in selected) { + val total = counts[PortabilityCategory.SUBSCRIPTIONS] ?: 0L + val processed = processPaged( + source, + PortabilityCategory.SUBSCRIPTIONS, + progress, + SUBSCRIPTION_BATCH_SIZE, + { true }, + ) { records -> commit(PortabilityCategory.SUBSCRIPTIONS, records, PortabilityImportStage.SUBSCRIPTIONS, total) } + if (processed == 0L) { + commit(PortabilityCategory.SUBSCRIPTIONS, emptyList(), PortabilityImportStage.SUBSCRIPTIONS, total) + } + } + + if (PortabilityCategory.HISTORY in selected) { + val total = counts[PortabilityCategory.HISTORY] ?: 0L + val recent = latestHistory(source, progress) + recentHistoryKeys += recent.map { it.stableKey() } + commit(PortabilityCategory.HISTORY, recent, PortabilityImportStage.RECENT_HISTORY, recent.size.toLong()) + remainingHistory = (total - recent.size).coerceAtLeast(0L) + } + + if (PortabilityCategory.PLAYLISTS in selected) { + val total = counts[PortabilityCategory.PLAYLISTS] ?: 0L + var processed = processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_BATCH_SIZE, + { it is PortabilityPlaylist }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + processed += processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_VIDEO_BATCH_SIZE, + { it is PortabilityPlaylistVideo }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + processed += processPaged( + source, + PortabilityCategory.PLAYLISTS, + progress, + PLAYLIST_VIDEO_BATCH_SIZE, + { it !is PortabilityPlaylist && it !is PortabilityPlaylistVideo }, + ) { records -> commit(PortabilityCategory.PLAYLISTS, records, PortabilityImportStage.PLAYLISTS, total) } + if (processed == 0L) { + commit(PortabilityCategory.PLAYLISTS, emptyList(), PortabilityImportStage.PLAYLISTS, total) + } + } + + if (PortabilityCategory.HISTORY in selected && remainingHistory > 0L) { + processPaged( + source, + PortabilityCategory.HISTORY, + progress, + HISTORY_BATCH_SIZE, + { it !is PortabilityHistory || it.stableKey() !in recentHistoryKeys }, + ) { records -> + commit(PortabilityCategory.HISTORY, records, PortabilityImportStage.HISTORY, remainingHistory) + } + } + + selected + .filterNot { + it == PortabilityCategory.SUBSCRIPTIONS || + it == PortabilityCategory.HISTORY || + it == PortabilityCategory.PLAYLISTS + } + .sortedBy(PortabilityCategory::wireName) + .forEach { category -> + progress.setStage(PortabilityImportStage.REMAINING, category, counts[category]) + dataPort.import( + userId, + source, + PortabilityImportRequest(setOf(category), request.duplicatePolicy), + onCategoryProgress = { _, count -> progress.add(count) }, + onCategoryComplete = { _, imported -> + result[category.wireName] = imported + onCommitted(result.toMap()) + progress.checkpoint() + }, + ) + } + return result + } + + private suspend fun processPaged( + source: PortabilityRecordSource, + category: PortabilityCategory, + progress: PortabilityProgressReporter, + batchSize: Int, + include: (PortabilityRecord) -> Boolean, + commit: suspend (List) -> Unit, + ): Long { + var cursor: Long? = null + var matched = 0L + val pending = ArrayList(batchSize) + while (true) { + progress.ensureActive() + val page = source.readBatch(category, cursor, READ_PAGE_SIZE) + if (page.records.isEmpty()) break + check(page.nextCursor != null && page.nextCursor != cursor) { "Portability source cursor did not advance" } + cursor = page.nextCursor + page.records.forEach { record -> + if (include(record)) { + matched++ + pending += record + if (pending.size == batchSize) { + commit(pending.toList()) + pending.clear() + } + } + } + } + if (pending.isNotEmpty()) commit(pending) + return matched + } + + private fun latestHistory( + source: PortabilityRecordSource, + progress: PortabilityProgressReporter, + ): List { + val oldestFirst = compareBy({ it.watchedAt }, { it.stableKey() }) + val recent = PriorityQueue(RECENT_HISTORY_LIMIT, oldestFirst) + source.forEach(PortabilityCategory.HISTORY) { record -> + progress.ensureActive() + if (record !is PortabilityHistory) return@forEach + if (recent.size < RECENT_HISTORY_LIMIT) recent += record + else if (oldestFirst.compare(record, recent.peek()) > 0) { + recent.remove() + recent += record + } + } + return recent.sortedWith(compareByDescending { it.watchedAt }.thenBy { it.stableKey() }) + } + + private companion object { + const val READ_PAGE_SIZE = 500 + const val SUBSCRIPTION_BATCH_SIZE = 50 + const val RECENT_HISTORY_LIMIT = 20 + const val HISTORY_BATCH_SIZE = 500 + const val PLAYLIST_BATCH_SIZE = 5 + const val PLAYLIST_VIDEO_BATCH_SIZE = 50 + } +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt new file mode 100644 index 00000000..72f31a1f --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/TypeTypePortabilityCoreImportTest.kt @@ -0,0 +1,58 @@ +package dev.typetype.server.portability + +import dev.typetype.server.TEST_USER_ID +import dev.typetype.server.TestDatabase +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.PlaylistVideosTable +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class TypeTypePortabilityCoreImportTest { + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `playlist progress counts each source record once`() = runBlocking { + val source = PortabilityBatchRecordSource( + PortabilityCategory.PLAYLISTS, + listOf( + PortabilityPlaylist("favorites", "Favorites"), + PortabilityPlaylistVideo("favorites", 0, video()), + ), + ) + var processed = 0 + + val imported = DatabaseFactory.query { + TypeTypePortabilityCoreImport.write( + TEST_USER_ID, + PortabilityCategory.PLAYLISTS, + source, + PortabilityDuplicatePolicy.SKIP, + ) { processed++ } + } + + assertEquals(2L, imported) + assertEquals(2, processed) + assertEquals(1L, transaction { PlaylistVideosTable.selectAll().count() }) + } + + private fun video() = PortabilityVideo( + url = "https://youtube.com/watch?v=playlist-test", + title = "Playlist video", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://youtube.com/channel/channel", + ) +} diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt new file mode 100644 index 00000000..28cf301e --- /dev/null +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutImportWorkflowTest.kt @@ -0,0 +1,141 @@ +package dev.typetype.server.portability + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.nio.file.Path + +class YoutubeTakeoutImportWorkflowTest { + @TempDir + lateinit var directory: Path + + @Test + fun `imports subscriptions then recent history and playlists in committed batches`() = runBlocking { + val source = PortabilitySpool.create(directory) + try { + repeat(125) { source.write(PortabilitySubscription("https://youtube.com/channel/UC$it")) } + repeat(1_025) { index -> source.write(history(index)) } + source.write(PortabilityPlaylist("favorites", "Favorites")) + repeat(55) { index -> + source.write( + PortabilityPlaylistVideo( + "favorites", + index, + video(index), + ), + ) + } + + val dataPort = RecordingDataPort() + val job = PortabilityJob( + id = "takeout-workflow", + ownerId = "owner", + kind = PortabilityJobKind.IMPORT, + directory = directory, + requestId = null, + clock = System::currentTimeMillis, + ) + val progress = PortabilityProgressReporter( + job, + PortabilityProgressPhase.APPLYING, + PortabilityProgressUnit.RECORDS, + total = 1_206L, + interval = 100L, + ) + var partial = emptyMap() + + val result = YoutubeTakeoutImportWorkflow(dataPort).apply( + "owner", + source, + PortabilityImportRequest( + setOf(PortabilityCategory.SUBSCRIPTIONS, PortabilityCategory.HISTORY, PortabilityCategory.PLAYLISTS), + PortabilityDuplicatePolicy.REPLACE, + ), + progress, + ) { + partial = it + job.updateResult(it) + } + + assertEquals( + listOf( + PortabilityCategory.SUBSCRIPTIONS to 50, + PortabilityCategory.SUBSCRIPTIONS to 50, + PortabilityCategory.SUBSCRIPTIONS to 25, + PortabilityCategory.HISTORY to 20, + PortabilityCategory.PLAYLISTS to 1, + PortabilityCategory.PLAYLISTS to 50, + PortabilityCategory.PLAYLISTS to 5, + PortabilityCategory.HISTORY to 500, + PortabilityCategory.HISTORY to 500, + PortabilityCategory.HISTORY to 5, + ), + dataPort.batches.map { it.category to it.records.size }, + ) + assertEquals((1_024 downTo 1_005).map(Int::toLong), dataPort.batches[3].records.map { (it as PortabilityHistory).watchedAt }) + assertEquals((0 until 500).map(Int::toLong), dataPort.batches[7].records.map { (it as PortabilityHistory).watchedAt }) + assertEquals(125L, result["subscriptions"]) + assertEquals(1_025L, result["history"]) + assertEquals(56L, result["playlists"]) + assertEquals(result, partial) + assertEquals(result, job.snapshot().result) + assertEquals(1_206L, job.snapshot().progress?.processed) + assertEquals(1_206L, job.snapshot().progress?.total) + assertEquals(1_005L, job.snapshot().progress?.stageProcessed) + assertEquals(1_005L, job.snapshot().progress?.stageTotal) + assertEquals(10L, job.snapshot().progress?.checkpoint) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches.first().policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[1].policy) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches[3].policy) + assertEquals(PortabilityDuplicatePolicy.REPLACE, dataPort.batches[4].policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[5].policy) + assertEquals(PortabilityDuplicatePolicy.SKIP, dataPort.batches[7].policy) + } finally { + source.delete() + } + } + + private fun history(index: Int) = PortabilityHistory(video(index), index.toLong()) + + private fun video(index: Int) = PortabilityVideo( + url = "https://youtube.com/watch?v=$index", + title = "Video $index", + thumbnailUrl = "", + durationSeconds = 60L, + channelName = "Channel", + channelUrl = "https://youtube.com/channel/channel", + ) +} + +private class RecordingDataPort : PortabilityDataPort { + val batches = mutableListOf() + + override suspend fun import( + userId: String, + source: PortabilityRecordSource, + request: PortabilityImportRequest, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + onCategoryProgress: (PortabilityCategory, Long) -> Unit, + ): Map { + val category = request.categories.single() + val records = buildList { source.forEach(category, ::add) } + batches += Batch(category, records, request.duplicatePolicy) + onCategoryProgress(category, records.size.toLong()) + onCategoryComplete(category, records.size.toLong()) + return mapOf(category.wireName to records.size.toLong()) + } + + override suspend fun export( + userId: String, + categories: Set, + sink: PortabilityRecordSink, + onCategoryComplete: (PortabilityCategory, Long) -> Unit, + ) = Unit + + data class Batch( + val category: PortabilityCategory, + val records: List, + val policy: PortabilityDuplicatePolicy, + ) +} From fe2e64c2776f2c484815b32bb0e107f728d95b7f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 10:03:37 +0000 Subject: [PATCH 119/137] chore: bump server to 1.9.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 5cc6396e..3daee93a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.8.2 +appVersion=1.9.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From 5a52dfd2fda86dbb935643bce3d0a235c8995d60 Mon Sep 17 00:00:00 2001 From: tam1m Date: Sat, 26 Sep 2026 11:34:00 +0200 Subject: [PATCH 120/137] fix: decode extractor responses with declared charset --- .../OkHttpExtractorResponseMapper.kt | 3 +- .../OkHttpExtractorResponseMapperTest.kt | 81 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt diff --git a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt index 9a47f893..0e770616 100644 --- a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt +++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt @@ -4,8 +4,9 @@ import org.schabi.newpipe.extractor.downloader.Response object OkHttpExtractorResponseMapper { fun toExtractorResponse(httpResponse: okhttp3.Response): Response { + val contentType = httpResponse.body.contentType() val responseBodyBytes = httpResponse.body.bytes() - val responseBody = responseBodyBytes.toString(Charsets.UTF_8) + val responseBody = responseBodyBytes.toString(contentType?.charset() ?: Charsets.UTF_8) return Response( httpResponse.code, httpResponse.message, diff --git a/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt new file mode 100644 index 00000000..b1bb99c1 --- /dev/null +++ b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt @@ -0,0 +1,81 @@ +package dev.typetype.server.downloader + +import java.nio.charset.Charset +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class OkHttpExtractorResponseMapperTest { + + private fun httpResponse(contentType: String?, bytes: ByteArray): okhttp3.Response = + okhttp3.Response.Builder() + .request(Request.Builder().url("https://example.org/complete/search").build()) + .protocol(Protocol.HTTP_2) + .code(200) + .message("OK") + .body(bytes.toResponseBody(contentType?.toMediaType())) + .build() + + @Test + fun `decodes iso-8859-1 accented characters`() { + val expected = "ä ö ü ß ë Ë é è ê ç à ñ" + val bytes = expected.toByteArray(Charsets.ISO_8859_1) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=ISO-8859-1", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + assertArrayEquals(bytes, mapped.rawResponseBody()) + } + + @Test + fun `decodes utf-8 body when charset is declared`() { + val expected = "ä ö ü ß 🎵" + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("application/json; charset=utf-8", expected.toByteArray(Charsets.UTF_8)), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `defaults to utf-8 without content type`() { + val expected = "ä ö ü ß 🎵" + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse(null, expected.toByteArray(Charsets.UTF_8)), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `decodes shift_jis body used for japanese locales`() { + val expected = "あいう" + val bytes = expected.toByteArray(Charset.forName("Shift_JIS")) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=Shift_JIS", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + } + + @Test + fun `decodes windows-1251 body used for cyrillic locales`() { + val expected = "А Б В" + val bytes = expected.toByteArray(Charset.forName("windows-1251")) + + val mapped = OkHttpExtractorResponseMapper.toExtractorResponse( + httpResponse("text/javascript; charset=windows-1251", bytes), + ) + + assertEquals(expected, mapped.responseBody()) + } +} From 395f9ab7f692ed9e33205038951a3c4e40fbcf97 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 14:38:47 +0000 Subject: [PATCH 121/137] fix: type YouTube extraction failures --- .../server/models/ExtractionResult.kt | 4 ++ .../server/services/ExtractionResilience.kt | 16 +++--- .../services/StreamExtractionErrorMapper.kt | 16 ++++++ .../server/ExtractionResilienceTest.kt | 47 +++++++++++++++++ .../server/StreamExtractionErrorMapperTest.kt | 52 +++++++++++++++++-- 5 files changed, 122 insertions(+), 13 deletions(-) diff --git a/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt b/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt index 01313a9a..2e9b2d8e 100644 --- a/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt +++ b/server-core/src/main/kotlin/dev/typetype/server/models/ExtractionResult.kt @@ -13,4 +13,8 @@ sealed class ExtractionResult { enum class ExtractionFailureKind { Unknown, YoutubeSessionRejected, + LiveEventNotStarted, + ScheduledPremiere, + ContentUnavailable, + ProviderAccessBlocked, } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt index 6c806b75..47bf7438 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt @@ -1,11 +1,11 @@ package dev.typetype.server.services +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay -import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException -import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException import org.schabi.newpipe.extractor.exceptions.NeedLoginException -import org.schabi.newpipe.extractor.exceptions.PaidContentException -import org.schabi.newpipe.extractor.exceptions.PrivateContentException +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException suspend fun withExtractionRetry( attempts: Int = 3, @@ -27,11 +27,11 @@ suspend fun withExtractionRetry( } private fun Throwable.isRetriableExtractionError(): Boolean = when (this) { - is GeographicRestrictionException, - is PaidContentException, + is CancellationException, + is ContentNotAvailableException, is NeedLoginException, - is AgeRestrictedContentException, - is PrivateContentException, + is AntiBotException, + is ReCaptchaException, is IllegalArgumentException -> false else -> true } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt index ac1cc565..7bd91b79 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/StreamExtractionErrorMapper.kt @@ -4,7 +4,9 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ExtractionFailureKind import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException +import org.schabi.newpipe.extractor.exceptions.LiveNotStartException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException @@ -13,6 +15,7 @@ import org.schabi.newpipe.extractor.exceptions.VideoNotReleaseException import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException object StreamExtractionErrorMapper { + const val UPSTREAM_FAILURE_CODE = "upstream_failure" const val MEMBERS_ONLY_FALLBACK = "This video is only available for members" const val PAID_CONTENT_FALLBACK = "This video is a paid video" const val GEOGRAPHIC_RESTRICTION_CODE = "geographic_restriction" @@ -37,9 +40,15 @@ object StreamExtractionErrorMapper { sanitize(error.message) ?: PAID_CONTENT_FALLBACK, "paid_content", ) + is LiveNotStartException -> ExtractionResult.Failure( + sanitize(error.message) ?: "This live event has not started yet", + "live_not_started", + ExtractionFailureKind.LiveEventNotStarted, + ) is VideoNotReleaseException -> ExtractionResult.Failure( sanitize(error.message) ?: "This premiere has not started yet", "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, ) is AgeRestrictedContentException -> ExtractionResult.BadRequest( sanitize(error.message) ?: "This video is age-restricted", @@ -57,9 +66,16 @@ object StreamExtractionErrorMapper { is ReCaptchaException -> ExtractionResult.Failure( PROVIDER_ACCESS_BLOCKED_FALLBACK, PROVIDER_ACCESS_BLOCKED_CODE, + ExtractionFailureKind.ProviderAccessBlocked, + ) + is ContentNotAvailableException -> ExtractionResult.Failure( + sanitize(error.message) ?: "This content is not available", + "content_unavailable", + ExtractionFailureKind.ContentUnavailable, ) else -> ExtractionResult.Failure( sanitize(error.message) ?: fallback, + if (error.isYoutubeSessionRejected()) "youtube_session_rejected" else UPSTREAM_FAILURE_CODE, kind = if (error.isYoutubeSessionRejected()) { ExtractionFailureKind.YoutubeSessionRejected } else { diff --git a/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt index 6572e686..1732f0db 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt @@ -1,9 +1,13 @@ package dev.typetype.server import dev.typetype.server.services.withExtractionRetry +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertSame import org.junit.jupiter.api.Test +import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException class ExtractionResilienceTest { @Test @@ -41,4 +45,47 @@ class ExtractionResilienceTest { assertEquals(IllegalArgumentException::class.java, error?.javaClass) assertEquals(1, calls) } + + @Test + fun `does not retry content availability failures`() = runBlocking { + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw ContentNotAvailableException("unavailable") + } + }.exceptionOrNull() + + assertEquals(ContentNotAvailableException::class.java, error?.javaClass) + assertEquals(1, calls) + } + + @Test + fun `does not retry provider bot challenges`() = runBlocking { + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw AntiBotException("challenge") + } + }.exceptionOrNull() + + assertEquals(AntiBotException::class.java, error?.javaClass) + assertEquals(1, calls) + } + + @Test + fun `propagates cancellation without retry`() = runBlocking { + val cancelled = CancellationException("cancelled") + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw cancelled + } + }.exceptionOrNull() + + assertSame(cancelled, error) + assertEquals(1, calls) + } } diff --git a/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt b/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt index ad282d34..1b3d44d5 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/StreamExtractionErrorMapperTest.kt @@ -9,7 +9,9 @@ import org.junit.jupiter.api.Assumptions.assumeTrue import org.junit.jupiter.api.Test import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.AntiBotException +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException +import org.schabi.newpipe.extractor.exceptions.LiveNotStartException import org.schabi.newpipe.extractor.exceptions.NeedLoginException import org.schabi.newpipe.extractor.exceptions.PaidContentException import org.schabi.newpipe.extractor.exceptions.PrivateContentException @@ -49,14 +51,49 @@ class StreamExtractionErrorMapperTest { @Test fun `maps upcoming premieres to a stable availability code`() { val result = StreamExtractionErrorMapper.map(VideoNotReleaseException("Premieres in 200 days")) - assertEquals(ExtractionResult.Failure("Premieres in 200 days", "scheduled_premiere"), result) + assertEquals( + ExtractionResult.Failure( + "Premieres in 200 days", + "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, + ), + result, + ) + } + + @Test + fun `maps upcoming live events to a stable availability code`() { + val message = "This live event will begin in 20 minutes." + val result = StreamExtractionErrorMapper.map(LiveNotStartException(message)) + + assertEquals( + ExtractionResult.Failure(message, "live_not_started", ExtractionFailureKind.LiveEventNotStarted), + result, + ) + } + + @Test + fun `maps generic content unavailability to a stable kind`() { + val message = "This content is not available" + val result = StreamExtractionErrorMapper.map(ContentNotAvailableException(message)) + + assertEquals( + ExtractionResult.Failure(message, "content_unavailable", ExtractionFailureKind.ContentUnavailable), + result, + ) } @Test fun `does not rewrite youtube timeout message to members-only fallback`() { val timeout = IllegalStateException("Error occurs when fetching the page. Try increase the loading timeout in Settings.") val mapped = StreamExtractionErrorMapper.map(timeout, sourceUrl = "https://www.youtube.com/watch?v=test") - assertEquals(ExtractionResult.Failure("Error occurs when fetching the page. Try increase the loading timeout in Settings."), mapped) + assertEquals( + ExtractionResult.Failure( + "Error occurs when fetching the page. Try increase the loading timeout in Settings.", + StreamExtractionErrorMapper.UPSTREAM_FAILURE_CODE, + ), + mapped, + ) } @Test @@ -88,6 +125,7 @@ class StreamExtractionErrorMapperTest { val expected = ExtractionResult.Failure( StreamExtractionErrorMapper.PROVIDER_ACCESS_BLOCKED_FALLBACK, "provider_access_blocked", + ExtractionFailureKind.ProviderAccessBlocked, ) assertEquals(expected, antiBot) assertEquals(expected, captcha) @@ -106,8 +144,10 @@ class StreamExtractionErrorMapperTest { fun `maps unknown exceptions to failure`() { val result = StreamExtractionErrorMapper.map(IllegalStateException("boom")) assertTrue(result is ExtractionResult.Failure) - assertEquals("boom", (result as ExtractionResult.Failure).message) - assertEquals(ExtractionFailureKind.Unknown, result.kind) + val failure = result as ExtractionResult.Failure + assertEquals("boom", failure.message) + assertEquals(ExtractionFailureKind.Unknown, failure.kind) + assertEquals(StreamExtractionErrorMapper.UPSTREAM_FAILURE_CODE, failure.code) } @Test @@ -122,6 +162,8 @@ class StreamExtractionErrorMapperTest { val result = StreamExtractionErrorMapper.map(error) assertTrue(result is ExtractionResult.Failure) - assertEquals(ExtractionFailureKind.YoutubeSessionRejected, (result as ExtractionResult.Failure).kind) + val failure = result as ExtractionResult.Failure + assertEquals(ExtractionFailureKind.YoutubeSessionRejected, failure.kind) + assertEquals("youtube_session_rejected", failure.code) } } From e1de8baa0346722c7726744a076bf267bb504bb9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 14:38:51 +0000 Subject: [PATCH 122/137] fix: map YouTube extraction statuses --- openapi/components/common.yaml | 4 +- openapi/paths/streams.yaml | 114 +++++++++++++++++- .../server/routes/StreamDeliveryMode.kt | 2 + .../typetype/server/routes/StreamRoutes.kt | 18 ++- .../routes/YoutubeExtractionHttpStatus.kt | 24 ++++ .../dev/typetype/server/StreamRoutesTest.kt | 101 +++++++++++++++- 6 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt diff --git a/openapi/components/common.yaml b/openapi/components/common.yaml index 5294ca04..e6193b47 100644 --- a/openapi/components/common.yaml +++ b/openapi/components/common.yaml @@ -15,7 +15,9 @@ ErrorResponse: Stable machine-readable error code when available. Playback extraction routes may return geographic_restriction, provider_access_blocked, private_content, no_playable_streams, age_restricted, scheduled_premiere, - paid_content, members_only, or youtube_session_required. + live_not_started, content_unavailable, upstream_failure, + youtube_session_rejected, youtube_session_needs_reconnect, paid_content, + members_only, or youtube_session_required. requestId: { type: string, nullable: true } RequestIdHeader: description: Request correlation id returned by TypeType-Server. diff --git a/openapi/paths/streams.yaml b/openapi/paths/streams.yaml index b4aeb7a6..dc3d2aff 100644 --- a/openapi/paths/streams.yaml +++ b/openapi/paths/streams.yaml @@ -27,8 +27,42 @@ YoutubeSabrStreams: $ref: ../components/common.yaml#/JsonError '403': $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse YoutubeLiveHls: get: tags: [extraction] @@ -54,8 +88,42 @@ YoutubeLiveHls: $ref: ../components/common.yaml#/JsonError '403': $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse YoutubeSabrBootstrap: get: tags: [extraction] @@ -85,8 +153,42 @@ YoutubeSabrBootstrap: $ref: ../components/common.yaml#/JsonError '403': $ref: ../components/common.yaml#/JsonError - '422': - $ref: ../components/common.yaml#/JsonError + '404': + description: YouTube reports that this video is unavailable. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '409': + description: The live event or premiere has not started yet. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '502': + description: YouTube extraction failed or returned no playable stream. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse + '503': + description: YouTube is temporarily rejecting requests from this server. + headers: + X-Request-ID: + $ref: ../components/common.yaml#/RequestIdHeader + content: + application/json: + schema: + $ref: ../components/common.yaml#/ErrorResponse NicoNicoStreams: get: tags: [extraction] diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt index a8640f93..94beff2c 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamDeliveryMode.kt @@ -19,4 +19,6 @@ internal enum class StreamDeliveryMode { } fun isSabr(): Boolean = this == YoutubeSabr + + fun isYoutube(): Boolean = this == YoutubeSabr || this == YoutubeLiveHls } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt index c96da3af..7c472519 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/StreamRoutes.kt @@ -157,7 +157,9 @@ private fun Route.streamRoute( } if (!data.hasPlayableSource()) { return@get call.respond( - HttpStatusCode.UnprocessableEntity, + if (deliveryMode.isYoutube()) { + HttpStatusCode.BadGateway + } else HttpStatusCode.UnprocessableEntity, ErrorResponse( "No compatible stream is available for this video", "no_playable_streams", @@ -185,10 +187,16 @@ private fun Route.streamRoute( ) call.respond(publicData) } - is ExtractionResult.BadRequest -> - call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message, result.code)) - is ExtractionResult.Failure -> - call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message, result.code)) + is ExtractionResult.BadRequest -> call.respond( + if (deliveryMode.isYoutube()) youtubeBadRequestStatus(result.code) else HttpStatusCode.BadRequest, + ErrorResponse(result.message, result.code), + ) + is ExtractionResult.Failure -> call.respond( + if (deliveryMode.isYoutube()) { + youtubeExtractionFailureStatus(result) + } else HttpStatusCode.UnprocessableEntity, + ErrorResponse(result.message, result.code), + ) } } } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt new file mode 100644 index 00000000..9e2a92af --- /dev/null +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/YoutubeExtractionHttpStatus.kt @@ -0,0 +1,24 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ExtractionFailureKind +import dev.typetype.server.models.ExtractionResult +import io.ktor.http.HttpStatusCode + +internal fun youtubeExtractionFailureStatus(failure: ExtractionResult.Failure): HttpStatusCode = + when (failure.kind) { + ExtractionFailureKind.LiveEventNotStarted, + ExtractionFailureKind.ScheduledPremiere -> HttpStatusCode.Conflict + ExtractionFailureKind.ContentUnavailable -> HttpStatusCode.NotFound + ExtractionFailureKind.ProviderAccessBlocked -> HttpStatusCode.ServiceUnavailable + ExtractionFailureKind.Unknown, + ExtractionFailureKind.YoutubeSessionRejected -> HttpStatusCode.BadGateway + } + +internal fun youtubeBadRequestStatus(code: String): HttpStatusCode = when (code) { + "age_restricted", + "geographic_restriction", + "members_only", + "paid_content", + "private_content" -> HttpStatusCode.Forbidden + else -> HttpStatusCode.BadRequest +} diff --git a/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt index 4c231589..f5d160b6 100644 --- a/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt +++ b/server-http/src/test/kotlin/dev/typetype/server/StreamRoutesTest.kt @@ -1,6 +1,7 @@ package dev.typetype.server import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.ExtractionFailureKind import dev.typetype.server.routes.streamRoutes import dev.typetype.server.services.StreamService import io.ktor.client.request.get @@ -80,23 +81,111 @@ class StreamRoutesTest { val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=live") - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertEquals(HttpStatusCode.BadGateway, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } @Test - fun `GET streams returns 422 on Failure`() = testApplication { + fun `GET scheduled live events return a typed conflict`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns - ExtractionResult.Failure("Premieres in 200 days", "scheduled_premiere") + ExtractionResult.Failure( + "This live event will begin in 20 minutes.", + "live_not_started", + ExtractionFailureKind.LiveEventNotStarted, + ) application { install(ContentNegotiation) { json() } routing { streamRoutes(streamService) } } val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertEquals(HttpStatusCode.Conflict, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"live_not_started\"")) + } + + @Test + fun `GET scheduled premieres return a typed conflict`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "Premieres in 200 days", + "scheduled_premiere", + ExtractionFailureKind.ScheduledPremiere, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=premiere") + + assertEquals(HttpStatusCode.Conflict, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"scheduled_premiere\"")) } + @Test + fun `GET unavailable YouTube content returns not found`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "This video is not available", + "content_unavailable", + ExtractionFailureKind.ContentUnavailable, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=missing") + + assertEquals(HttpStatusCode.NotFound, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"content_unavailable\"")) + } + + @Test + fun `GET YouTube extraction failures return gateway errors`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure("Temporary provider error", "upstream_failure") + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=bad") + assertEquals(HttpStatusCode.BadGateway, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"upstream_failure\"")) + } + + @Test + fun `GET provider blocks return service unavailable`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure( + "The provider is blocking requests", + "provider_access_blocked", + ExtractionFailureKind.ProviderAccessBlocked, + ) + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=blocked") + + assertEquals(HttpStatusCode.ServiceUnavailable, response.status) + assertTrue(response.bodyAsText().contains("\"code\":\"provider_access_blocked\"")) + } + + @Test + fun `GET stream failure preserves non-YouTube status behavior`() = testApplication { + coEvery { streamService.getStreamInfo(any()) } returns + ExtractionResult.Failure("Provider failure", "upstream_failure") + application { + install(ContentNegotiation) { json() } + routing { streamRoutes(streamService) } + } + + val response = client.get("/streams/niconico?url=https://www.nicovideo.jp/watch/sm9") + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + } + @Test fun `GET streams returns 400 on BadRequest`() = testApplication { coEvery { streamService.getStreamInfo(any()) } returns @@ -106,7 +195,7 @@ class StreamRoutesTest { routing { streamRoutes(streamService) } } val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=paid") - assertEquals(HttpStatusCode.BadRequest, response.status) + assertEquals(HttpStatusCode.Forbidden, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"paid_content\"")) } @@ -172,7 +261,7 @@ class StreamRoutesTest { val response = client.get("/streams/youtube/sabr?url=https://youtube.com/watch?v=test") val body = response.bodyAsText() - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertEquals(HttpStatusCode.BadGateway, response.status) assertTrue(body.contains("\"code\":\"no_playable_streams\"")) } From af3fc81c9d855f06a20d8320e18bca0bdf60bc23 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 14:45:25 +0000 Subject: [PATCH 123/137] test: expect gateway status for no YouTube stream --- .../kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt index 0ee59409..c3d19480 100644 --- a/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt +++ b/src/test/kotlin/dev/typetype/server/StreamRoutesDeliveryModeTest.kt @@ -71,7 +71,7 @@ class StreamRoutesDeliveryModeTest { val response = client.get("/streams/youtube/sabr?url=$VIDEO_URL") - assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertEquals(HttpStatusCode.BadGateway, response.status) assertTrue(response.bodyAsText().contains("\"code\":\"no_playable_streams\"")) } From f8ea89ee7dd0998985c6f6c1348ac027c7691320 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 14:53:49 +0000 Subject: [PATCH 124/137] fix: preserve non-retriable restriction errors --- .../server/services/ExtractionResilience.kt | 8 +++++++ .../server/ExtractionResilienceTest.kt | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt index 47bf7438..3f4a0016 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ExtractionResilience.kt @@ -2,9 +2,13 @@ package dev.typetype.server.services import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay +import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException import org.schabi.newpipe.extractor.exceptions.AntiBotException import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException +import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException import org.schabi.newpipe.extractor.exceptions.NeedLoginException +import org.schabi.newpipe.extractor.exceptions.PaidContentException +import org.schabi.newpipe.extractor.exceptions.PrivateContentException import org.schabi.newpipe.extractor.exceptions.ReCaptchaException suspend fun withExtractionRetry( @@ -29,6 +33,10 @@ suspend fun withExtractionRetry( private fun Throwable.isRetriableExtractionError(): Boolean = when (this) { is CancellationException, is ContentNotAvailableException, + is AgeRestrictedContentException, + is GeographicRestrictionException, + is PaidContentException, + is PrivateContentException, is NeedLoginException, is AntiBotException, is ReCaptchaException, diff --git a/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt index 1732f0db..b01072df 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/ExtractionResilienceTest.kt @@ -88,4 +88,27 @@ class ExtractionResilienceTest { assertSame(cancelled, error) assertEquals(1, calls) } + + @Test + fun `does not retry access restriction failures`() = runBlocking { + val failures = listOf( + org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException("age restricted"), + org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException("geographic restriction"), + org.schabi.newpipe.extractor.exceptions.PaidContentException("paid content"), + org.schabi.newpipe.extractor.exceptions.PrivateContentException("private content"), + ) + + failures.forEach { failure -> + var calls = 0 + val error = runCatching { + withExtractionRetry(attempts = 3, initialDelayMs = 1) { + calls++ + throw failure + } + }.exceptionOrNull() + + assertSame(failure, error) + assertEquals(1, calls) + } + } } From b8278c2e5b22ffe047d74972a5e5a6cc0a2f6ff7 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 19:09:54 +0200 Subject: [PATCH 125/137] fix: ignore unrelated Takeout CSV metadata --- .../YoutubeTakeoutCsvPortabilityReader.kt | 9 +++++++- .../YoutubeTakeoutCsvSchemaDetectorTest.kt | 22 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt index 08e452d4..794653fc 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvPortabilityReader.kt @@ -29,7 +29,11 @@ object YoutubeTakeoutCsvPortabilityReader { .forEach { readSubscriptions(zip, it.entry, sink) } classified.filter { it.kind == YoutubeTakeoutCsvKind.PLAYLIST_CONTENT } .forEach { readPlaylistItems(zip, it.entry, sink) } - classified.filter { it.kind == YoutubeTakeoutCsvKind.OTHER && it.maybePortable } + classified.filter { + it.kind == YoutubeTakeoutCsvKind.OTHER && + it.maybePortable && + isExpectedImportCsv(it.entry.name) + } .forEach { classifiedEntry -> sink.issue( PortabilityIssue( @@ -147,4 +151,7 @@ object YoutubeTakeoutCsvPortabilityReader { private fun PortabilityRecordSink.invalid(category: PortabilityCategory, kind: String) = issue(PortabilityIssue(category, "invalid_takeout_row", "An invalid YouTube Takeout $kind row was skipped")) + private fun isExpectedImportCsv(name: String): Boolean = + YoutubeTakeoutSchemaHints.isSubscriptionText(name) || YoutubeTakeoutSchemaHints.isPlaylistText(name) + } diff --git a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt index c076429f..20e2e0e3 100644 --- a/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt +++ b/server-portability/src/test/kotlin/dev/typetype/server/portability/YoutubeTakeoutCsvSchemaDetectorTest.kt @@ -47,6 +47,28 @@ class YoutubeTakeoutCsvSchemaDetectorTest { } } + @Test + fun `ignores unrelated Takeout CSV files that contain channel URLs`() { + val archive = directory.resolve("takeout-comments.zip") + ZipOutputStream(Files.newOutputStream(archive)).use { output -> + output.entry( + "Takeout/YouTube/comments/comments.csv", + "Comment,Author channel URL\nHello,https://www.youtube.com/channel/UC123456789012\n", + ) + } + val input = PortabilityInputFactory.create(archive, archive.fileName.toString(), "application/zip") + val spool = PortabilitySpool.create(directory) + + try { + YoutubeTakeoutPortabilityAdapter().decode(input, spool) + + assertEquals(emptyMap(), spool.counts()) + assertTrue(spool.issues().isEmpty()) + } finally { + spool.delete() + } + } + private fun ZipOutputStream.entry(name: String, value: String) { putNextEntry(ZipEntry(name)) write(value.toByteArray()) From c2a7df2b6e127414a5598f0987b7903a029af784 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 19:17:01 +0200 Subject: [PATCH 126/137] fix: propagate imported subscription avatars --- .../PortabilityMediaAvatarPropagation.kt | 53 +++++++++++++++++++ .../TypeTypePortabilityDataPort.kt | 1 + .../SubscriptionsAvatarRepairServiceTest.kt | 41 ++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt new file mode 100644 index 00000000..f3875537 --- /dev/null +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt @@ -0,0 +1,53 @@ +package dev.typetype.server.portability + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.services.ChannelUrlCanonicalizer +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.neq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +object PortabilityMediaAvatarPropagation { + fun propagate(userId: String): Int { + val avatars = SubscriptionsTable.selectAll() + .where { + (SubscriptionsTable.userId eq userId) and + (SubscriptionsTable.avatarUrl neq "") + } + .limit(MAX_SUBSCRIPTIONS) + .map { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) to + it[SubscriptionsTable.avatarUrl] + } + .toMap() + return avatars.entries.sumOf { (channelUrl, avatarUrl) -> + HistoryTable.update({ + (HistoryTable.userId eq userId) and + (HistoryTable.channelUrl eq channelUrl) and + (HistoryTable.channelAvatar eq "") + }) { it[HistoryTable.channelAvatar] = avatarUrl } + + PlaylistVideosTable.update({ + (PlaylistVideosTable.userId eq userId) and + (PlaylistVideosTable.channelUrl eq channelUrl) and + (PlaylistVideosTable.channelAvatar eq "") + }) { it[PlaylistVideosTable.channelAvatar] = avatarUrl } + + WatchLaterTable.update({ + (WatchLaterTable.userId eq userId) and + (WatchLaterTable.channelUrl eq channelUrl) and + (WatchLaterTable.channelAvatar eq "") + }) { it[WatchLaterTable.channelAvatar] = avatarUrl } + + FavoritesTable.update({ + (FavoritesTable.userId eq userId) and + (FavoritesTable.channelUrl eq channelUrl) and + (FavoritesTable.channelAvatar eq "") + }) { it[FavoritesTable.channelAvatar] = avatarUrl } + } + } + + private const val MAX_SUBSCRIPTIONS = 2_000 +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt index 62bdb14c..fedd6e0a 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/TypeTypePortabilityDataPort.kt @@ -24,6 +24,7 @@ class TypeTypePortabilityDataPort : PortabilityDataPort { if (PortabilityCategory.SUBSCRIPTIONS in request.categories) { SubscriptionFeedCacheInvalidation.invalidate(userId) } + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(userId) } return result } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index de4341ef..1f2f20ab 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -6,6 +6,7 @@ import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.db.tables.WatchLaterTable import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.portability.PortabilityMediaAvatarPropagation import dev.typetype.server.services.SubscriptionAvatarRepairer import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService @@ -104,6 +105,28 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(WATCH_AVATAR_URL, repaired.last().avatarUrl) } + @Test + fun `propagates subscription avatars to imported history and collections`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = WATCH_AVATAR_URL + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "", watchedAt = 1L) + addWatchLater(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "") + addFavorite(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "") + + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(TEST_USER_ID) } + + assertEquals(WATCH_AVATAR_URL, storedHistoryAvatar()) + assertEquals(WATCH_AVATAR_URL, storedWatchLaterAvatar()) + assertEquals(WATCH_AVATAR_URL, storedFavoriteAvatar()) + } + private suspend fun addSubscription(channelUrl: String): Unit { service.add(TEST_USER_ID, SubscriptionItem(channelUrl = channelUrl, name = "Channel", avatarUrl = "")) } @@ -114,6 +137,24 @@ class SubscriptionsAvatarRepairServiceTest { }.single()[SubscriptionsTable.avatarUrl] } + private suspend fun storedHistoryAvatar(): String = DatabaseFactory.query { + HistoryTable.selectAll().where { + (HistoryTable.userId eq TEST_USER_ID) and (HistoryTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[HistoryTable.channelAvatar] + } + + private suspend fun storedWatchLaterAvatar(): String = DatabaseFactory.query { + WatchLaterTable.selectAll().where { + (WatchLaterTable.userId eq TEST_USER_ID) and (WatchLaterTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[WatchLaterTable.channelAvatar] + } + + private suspend fun storedFavoriteAvatar(): String = DatabaseFactory.query { + FavoritesTable.selectAll().where { + (FavoritesTable.userId eq TEST_USER_ID) and (FavoritesTable.channelUrl eq WATCH_CHANNEL_URL) + }.single()[FavoritesTable.channelAvatar] + } + private suspend fun addWatchLater(channelUrl: String, avatarUrl: String): Unit = DatabaseFactory.query { WatchLaterTable.insert { it[userId] = TEST_USER_ID; it[url] = "https://video.test/watch"; it[title] = "Video"; it[thumbnail] = "" From 3acf040e683958121a2f539fee4efc1b71246d08 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 19:29:21 +0200 Subject: [PATCH 127/137] feat: expose opt-in channel notification settings --- .../server/routes/PushNotificationRoutes.kt | 2 - .../ChannelNotificationPreferenceService.kt | 2 +- ...hannelNotificationPreferenceServiceTest.kt | 37 +++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt index abd8ac9d..ebde90f4 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PushNotificationRoutes.kt @@ -22,13 +22,11 @@ internal fun Route.pushNotificationRoutes( ) { get("/notifications/channel-preferences") { call.withJwtAuth(authService) { userId -> - if (!call.requirePushEnabled(service)) return@withJwtAuth call.respond(service.listPreferences(userId)) } } put("/notifications/channel-preferences") { call.withJwtAuth(authService) { userId -> - if (!call.requirePushEnabled(service)) return@withJwtAuth val request = runCatching { call.receive() }.getOrNull() ?: return@withJwtAuth call.respond( HttpStatusCode.BadRequest, diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt index f435fffc..3e4a2d60 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/ChannelNotificationPreferenceService.kt @@ -26,7 +26,7 @@ class ChannelNotificationPreferenceService( val row = stored[channelUrl] ChannelNotificationPreference( channelUrl, - row?.get(ChannelNotificationPreferencesTable.enabled) ?: true, + row?.get(ChannelNotificationPreferencesTable.enabled) ?: false, row?.get(ChannelNotificationPreferencesTable.updatedAt) ?: 0L, ) } diff --git a/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt b/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt new file mode 100644 index 00000000..543825e1 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ChannelNotificationPreferenceServiceTest.kt @@ -0,0 +1,37 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.ChannelNotificationPreferenceService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class ChannelNotificationPreferenceServiceTest { + private val subscriptions = SubscriptionsService() + private val preferences = ChannelNotificationPreferenceService(subscriptions) + + companion object { + private const val CHANNEL_URL = "https://www.youtube.com/channel/UCNotification" + + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `channel notifications are disabled until explicitly enabled`() = runTest { + subscriptions.add(TEST_USER_ID, SubscriptionItem(CHANNEL_URL, "Channel", "")) + + assertEquals(false, preferences.list(TEST_USER_ID).single().enabled) + + preferences.set(TEST_USER_ID, CHANNEL_URL, true) + + assertEquals(true, preferences.list(TEST_USER_ID).single().enabled) + } +} From 08a0da5ad1a560722d4c231be477c7db689b65c0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 20:10:04 +0200 Subject: [PATCH 128/137] fix: resolve history avatars from subscriptions --- .../server/services/HistoryService.kt | 54 +++++++++++++++---- .../SubscriptionsAvatarRepairServiceTest.kt | 19 +++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt index 4d065d51..f2109400 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/HistoryService.kt @@ -3,14 +3,17 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.HistoryItem import org.jetbrains.exposed.v1.core.LowerCase import org.jetbrains.exposed.v1.core.SortOrder import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.greaterEq +import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.less import org.jetbrains.exposed.v1.core.like +import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.andWhere import org.jetbrains.exposed.v1.jdbc.batchInsert @@ -26,20 +29,49 @@ class HistoryService { .orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC) .toList() HistoryProgressMapper.toHistoryItemsForExport(userId, rows) + }.withSubscriptionAvatars(userId) + + suspend fun search(userId: String, q: String?, from: Long?, to: Long?, limit: Int, offset: Int): Pair, Long> { + val (items, total) = DatabaseFactory.query { + val query = HistoryTable.selectAll().where { HistoryTable.userId eq userId } + if (!q.isNullOrBlank()) { + val pattern = "%${q.lowercase()}%" + query.andWhere { (LowerCase(HistoryTable.title) like pattern) or (LowerCase(HistoryTable.channelName) like pattern) } + } + if (from != null) query.andWhere { HistoryTable.watchedAt greaterEq from } + if (to != null) query.andWhere { HistoryTable.watchedAt less to } + val total = query.count() + val rows = query.orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC).limit(limit).offset(offset.toLong()).toList() + HistoryProgressMapper.toHistoryItems(userId, rows) to total + } + return items.withSubscriptionAvatars(userId) to total } - suspend fun search(userId: String, q: String?, from: Long?, to: Long?, limit: Int, offset: Int): Pair, Long> = DatabaseFactory.query { - val query = HistoryTable.selectAll().where { HistoryTable.userId eq userId } - if (!q.isNullOrBlank()) { - val pattern = "%${q.lowercase()}%" - query.andWhere { (LowerCase(HistoryTable.title) like pattern) or (LowerCase(HistoryTable.channelName) like pattern) } + private suspend fun List.withSubscriptionAvatars(userId: String): List { + val missingUrls = filter { it.channelAvatar.isBlank() && it.channelUrl.isNotBlank() } + .map { ChannelUrlCanonicalizer.canonicalize(it.channelUrl) } + .distinct() + if (missingUrls.isEmpty()) return this + val avatars = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { + (SubscriptionsTable.userId eq userId) and + (SubscriptionsTable.channelUrl inList missingUrls) and + (SubscriptionsTable.avatarUrl neq "") + } + .map { + ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) to + it[SubscriptionsTable.avatarUrl] + } + .toMap() + } + if (avatars.isEmpty()) return this + return map { item -> + if (item.channelAvatar.isNotBlank()) item + else avatars[ChannelUrlCanonicalizer.canonicalize(item.channelUrl)]?.let { + item.copy(channelAvatar = it) + } ?: item } - if (from != null) query.andWhere { HistoryTable.watchedAt greaterEq from } - if (to != null) query.andWhere { HistoryTable.watchedAt less to } - val total = query.count() - val rows = query.orderBy(HistoryTable.watchedAt to SortOrder.DESC, HistoryTable.id to SortOrder.DESC).limit(limit).offset(offset.toLong()).toList() - val items = HistoryProgressMapper.toHistoryItems(userId, rows) - items to total } suspend fun add(userId: String, item: HistoryItem): HistoryItem = insert(userId, item, System.currentTimeMillis()) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index 1f2f20ab..44b134d6 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -12,6 +12,7 @@ import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.HistoryService import kotlinx.coroutines.test.runTest import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq @@ -127,6 +128,24 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(WATCH_AVATAR_URL, storedFavoriteAvatar()) } + @Test + fun `history reads subscription avatars without waiting for a repair write`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = WATCH_AVATAR_URL + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = "", watchedAt = 1L) + + val item = HistoryService().search(TEST_USER_ID, null, null, null, 1, 0).first.single() + + assertEquals(WATCH_AVATAR_URL, item.channelAvatar) + } + private suspend fun addSubscription(channelUrl: String): Unit { service.add(TEST_USER_ID, SubscriptionItem(channelUrl = channelUrl, name = "Channel", avatarUrl = "")) } From ff2dc78a66e4e054f7f9cede360d694dda7717fe Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 20:33:23 +0200 Subject: [PATCH 129/137] fix: repair imported subscription avatars --- .../services/SubscriptionAvatarRepairer.kt | 184 ++++++++++++++++++ .../PortabilityMediaAvatarPropagation.kt | 21 +- .../services/SubscriptionAvatarRepairer.kt | 102 ---------- .../SubscriptionsAvatarRepairServiceTest.kt | 19 ++ 4 files changed, 218 insertions(+), 108 deletions(-) create mode 100644 server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt delete mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt diff --git a/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt new file mode 100644 index 00000000..b2c3d585 --- /dev/null +++ b/server-domain/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt @@ -0,0 +1,184 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.FavoritesTable +import dev.typetype.server.db.tables.HistoryTable +import dev.typetype.server.db.tables.PlaylistVideosTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.models.SubscriptionItem +import org.jetbrains.exposed.v1.core.Column +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.neq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update + +object SubscriptionAvatarRepairer { + fun repair(userId: String, items: List): List { + val avatars = recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST, + maxSourceRows = MAX_AVATAR_SOURCE_ROWS, + ) + avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } + return items.withAvatars(avatars) + } + + fun repairImported(userId: String, items: List): List { + val avatars = recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_IMPORTED_AVATAR_REPAIR, + maxSourceRows = MAX_IMPORTED_AVATAR_SOURCE_ROWS, + ) + avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } + return items.withAvatars(avatars) + } + + fun resolve(userId: String, items: List): List = + items.withAvatars( + recoverableAvatars( + userId = userId, + items = items, + maxRepairs = MAX_AVATAR_REPAIR_PER_REQUEST, + maxSourceRows = MAX_AVATAR_SOURCE_ROWS, + ), + ) + + private fun recoverableAvatars( + userId: String, + items: List, + maxRepairs: Int, + maxSourceRows: Int, + ): Map { + val candidateUrls = items.filter { it.avatarUrl.isBlank() } + .map { it.channelUrl } + .distinct() + if (candidateUrls.isEmpty()) return emptyMap() + val avatars = knownAvatars(userId, candidateUrls, maxSourceRows) + .entries + .take(maxRepairs) + .associate { it.toPair() } + return avatars + } + + private fun List.withAvatars(avatars: Map): List = + if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } + + private fun knownAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map { + val avatars = linkedMapOf() + historyAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + playlistAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + watchLaterAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + favoriteAvatars(userId, channelUrls, maxSourceRows).forEach { avatars.putIfAbsent(it.key, it.value) } + return avatars + } + + private fun historyAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = HistoryTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + HistoryTable.userId, + HistoryTable.channelUrl, + HistoryTable.channelAvatar, + ) + } + .orderBy(HistoryTable.watchedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(HistoryTable.channelUrl, HistoryTable.channelAvatar) + + private fun playlistAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = PlaylistVideosTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + PlaylistVideosTable.userId, + PlaylistVideosTable.channelUrl, + PlaylistVideosTable.channelAvatar, + ) + } + .limit(maxSourceRows) + .associateAvatarRows(PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) + + private fun watchLaterAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = WatchLaterTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + WatchLaterTable.userId, + WatchLaterTable.channelUrl, + WatchLaterTable.channelAvatar, + ) + } + .orderBy(WatchLaterTable.addedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) + + private fun favoriteAvatars( + userId: String, + channelUrls: List, + maxSourceRows: Int, + ): Map = FavoritesTable.selectAll() + .where { + avatarSourceFilter( + userId, + channelUrls, + FavoritesTable.userId, + FavoritesTable.channelUrl, + FavoritesTable.channelAvatar, + ) + } + .orderBy(FavoritesTable.favoritedAt to SortOrder.DESC) + .limit(maxSourceRows) + .associateAvatarRows(FavoritesTable.channelUrl, FavoritesTable.channelAvatar) + + private fun updateAvatar(userId: String, channelUrl: String, avatarUrl: String): Int = SubscriptionsTable.update({ + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) and (SubscriptionsTable.avatarUrl eq "") + }) { + it[SubscriptionsTable.avatarUrl] = avatarUrl + } + + private fun avatarSourceFilter( + userId: String, + channelUrls: List, + userColumn: Column, + urlColumn: Column, + avatarColumn: Column, + ) = (userColumn eq userId) and (urlColumn inList channelUrls) and (avatarColumn neq "") + + private fun Iterable.associateAvatarRows( + urlColumn: Column, + avatarColumn: Column, + ): Map = mapNotNull { row -> + val avatar = row[avatarColumn].trim() + if (avatar.isProxyableAvatar()) ChannelUrlCanonicalizer.canonicalize(row[urlColumn]) to avatar else null + }.distinctBy { it.first }.toMap() + + private fun String.isProxyableAvatar(): Boolean = startsWith("https://") || startsWith("http://") + + private const val MAX_AVATAR_REPAIR_PER_REQUEST = 25 + private const val MAX_AVATAR_SOURCE_ROWS = 100 + private const val MAX_IMPORTED_AVATAR_REPAIR = 5_000 + private const val MAX_IMPORTED_AVATAR_SOURCE_ROWS = 20_000 +} diff --git a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt index f3875537..0eca3eed 100644 --- a/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt +++ b/server-portability/src/main/kotlin/dev/typetype/server/portability/PortabilityMediaAvatarPropagation.kt @@ -5,7 +5,9 @@ import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.db.tables.WatchLaterTable +import dev.typetype.server.models.SubscriptionItem import dev.typetype.server.services.ChannelUrlCanonicalizer +import dev.typetype.server.services.SubscriptionAvatarRepairer import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.neq @@ -14,17 +16,24 @@ import org.jetbrains.exposed.v1.jdbc.update object PortabilityMediaAvatarPropagation { fun propagate(userId: String): Int { - val avatars = SubscriptionsTable.selectAll() + val subscriptions = SubscriptionsTable.selectAll() .where { - (SubscriptionsTable.userId eq userId) and - (SubscriptionsTable.avatarUrl neq "") + SubscriptionsTable.userId eq userId } .limit(MAX_SUBSCRIPTIONS) .map { - ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) to - it[SubscriptionsTable.avatarUrl] + SubscriptionItem( + channelUrl = it[SubscriptionsTable.channelUrl], + name = it[SubscriptionsTable.name], + avatarUrl = it[SubscriptionsTable.avatarUrl], + subscribedAt = it[SubscriptionsTable.subscribedAt], + ) + } + val avatars = SubscriptionAvatarRepairer.repairImported(userId, subscriptions) + .filter { it.avatarUrl.isNotBlank() } + .associate { + ChannelUrlCanonicalizer.canonicalize(it.channelUrl) to it.avatarUrl } - .toMap() return avatars.entries.sumOf { (channelUrl, avatarUrl) -> HistoryTable.update({ (HistoryTable.userId eq userId) and diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt deleted file mode 100644 index 23731819..00000000 --- a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarRepairer.kt +++ /dev/null @@ -1,102 +0,0 @@ -package dev.typetype.server.services - -import dev.typetype.server.db.tables.FavoritesTable -import dev.typetype.server.db.tables.HistoryTable -import dev.typetype.server.db.tables.PlaylistVideosTable -import dev.typetype.server.db.tables.SubscriptionsTable -import dev.typetype.server.db.tables.WatchLaterTable -import dev.typetype.server.models.SubscriptionItem -import org.jetbrains.exposed.v1.core.Column -import org.jetbrains.exposed.v1.core.ResultRow -import org.jetbrains.exposed.v1.core.SortOrder -import org.jetbrains.exposed.v1.core.and -import org.jetbrains.exposed.v1.core.eq -import org.jetbrains.exposed.v1.core.inList -import org.jetbrains.exposed.v1.core.neq -import org.jetbrains.exposed.v1.jdbc.selectAll -import org.jetbrains.exposed.v1.jdbc.update - -object SubscriptionAvatarRepairer { - fun repair(userId: String, items: List): List { - val avatars = recoverableAvatars(userId, items) - avatars.forEach { (channelUrl, avatarUrl) -> updateAvatar(userId, channelUrl, avatarUrl) } - return items.withAvatars(avatars) - } - - fun resolve(userId: String, items: List): List = - items.withAvatars(recoverableAvatars(userId, items)) - - private fun recoverableAvatars(userId: String, items: List): Map { - val candidateUrls = items.filter { it.avatarUrl.isBlank() } - .map { it.channelUrl } - .distinct() - if (candidateUrls.isEmpty()) return emptyMap() - val avatars = knownAvatars(userId = userId, channelUrls = candidateUrls) - .entries - .take(MAX_AVATAR_REPAIR_PER_REQUEST) - .associate { it.toPair() } - return avatars - } - - private fun List.withAvatars(avatars: Map): List = - if (avatars.isEmpty()) this else map { item -> avatars[item.channelUrl]?.let { item.copy(avatarUrl = it) } ?: item } - - private fun knownAvatars(userId: String, channelUrls: List): Map { - val avatars = linkedMapOf() - historyAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - playlistAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - watchLaterAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - favoriteAvatars(userId, channelUrls).forEach { avatars.putIfAbsent(it.key, it.value) } - return avatars - } - - private fun historyAvatars(userId: String, channelUrls: List): Map = HistoryTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, HistoryTable.userId, HistoryTable.channelUrl, HistoryTable.channelAvatar) } - .orderBy(HistoryTable.watchedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(HistoryTable.channelUrl, HistoryTable.channelAvatar) - - private fun playlistAvatars(userId: String, channelUrls: List): Map = PlaylistVideosTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, PlaylistVideosTable.userId, PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) } - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(PlaylistVideosTable.channelUrl, PlaylistVideosTable.channelAvatar) - - private fun watchLaterAvatars(userId: String, channelUrls: List): Map = WatchLaterTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, WatchLaterTable.userId, WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) } - .orderBy(WatchLaterTable.addedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(WatchLaterTable.channelUrl, WatchLaterTable.channelAvatar) - - private fun favoriteAvatars(userId: String, channelUrls: List): Map = FavoritesTable.selectAll() - .where { avatarSourceFilter(userId, channelUrls, FavoritesTable.userId, FavoritesTable.channelUrl, FavoritesTable.channelAvatar) } - .orderBy(FavoritesTable.favoritedAt to SortOrder.DESC) - .limit(MAX_AVATAR_SOURCE_ROWS) - .associateAvatarRows(FavoritesTable.channelUrl, FavoritesTable.channelAvatar) - - private fun updateAvatar(userId: String, channelUrl: String, avatarUrl: String): Int = SubscriptionsTable.update({ - (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) and (SubscriptionsTable.avatarUrl eq "") - }) { - it[SubscriptionsTable.avatarUrl] = avatarUrl - } - - private fun avatarSourceFilter( - userId: String, - channelUrls: List, - userColumn: Column, - urlColumn: Column, - avatarColumn: Column, - ) = (userColumn eq userId) and (urlColumn inList channelUrls) and (avatarColumn neq "") - - private fun Iterable.associateAvatarRows( - urlColumn: Column, - avatarColumn: Column, - ): Map = mapNotNull { row -> - val avatar = row[avatarColumn].trim() - if (avatar.isProxyableAvatar()) ChannelUrlCanonicalizer.canonicalize(row[urlColumn]) to avatar else null - }.distinctBy { it.first }.toMap() - - private fun String.isProxyableAvatar(): Boolean = startsWith("https://") || startsWith("http://") - - private const val MAX_AVATAR_REPAIR_PER_REQUEST = 25 - private const val MAX_AVATAR_SOURCE_ROWS = 100 -} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt index 44b134d6..f2cb6e0d 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionsAvatarRepairServiceTest.kt @@ -128,6 +128,25 @@ class SubscriptionsAvatarRepairServiceTest { assertEquals(WATCH_AVATAR_URL, storedFavoriteAvatar()) } + @Test + fun `import propagation repairs subscription avatars from media first`() = runTest { + DatabaseFactory.query { + SubscriptionsTable.insert { + it[userId] = TEST_USER_ID + it[channelUrl] = WATCH_CHANNEL_URL + it[name] = "Channel" + it[avatarUrl] = "" + it[subscribedAt] = 1L + } + } + addHistory(channelUrl = WATCH_CHANNEL_URL, avatarUrl = WATCH_AVATAR_URL, watchedAt = 1L) + + DatabaseFactory.query { PortabilityMediaAvatarPropagation.propagate(TEST_USER_ID) } + + assertEquals(WATCH_AVATAR_URL, storedAvatar(WATCH_CHANNEL_URL)) + assertEquals(WATCH_AVATAR_URL, storedHistoryAvatar()) + } + @Test fun `history reads subscription avatars without waiting for a repair write`() = runTest { DatabaseFactory.query { From 8823cbf5ab9ab4b26efdc1c49696e7e76c89ff46 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 21:03:18 +0200 Subject: [PATCH 130/137] fix: stabilize playlist summaries and metadata repair --- .../server/services/PlaylistService.kt | 17 +++++++++- .../services/PlaylistVideoProgressMapper.kt | 6 +++- .../UserVideoMetadataRepairService.kt | 8 ++--- .../server/services/VideoMetadataResolver.kt | 2 +- .../typetype/server/PlaylistServiceTest.kt | 31 ++++++++++++++++--- .../UserVideoMetadataRepairRoutesTest.kt | 6 ++-- 6 files changed, 56 insertions(+), 14 deletions(-) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt index df2e9ad5..fe4c7e15 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistService.kt @@ -29,8 +29,23 @@ class PlaylistService { .where { PlaylistVideosTable.userId eq userId } .groupBy(PlaylistVideosTable.playlistId) .associate { row -> row[PlaylistVideosTable.playlistId] to row[videoCount].toInt() } + val firstVideos = PlaylistVideosTable + .selectAll() + .withDistinctOn(PlaylistVideosTable.playlistId) + .where { PlaylistVideosTable.userId eq userId } + .orderBy( + PlaylistVideosTable.playlistId to SortOrder.ASC, + PlaylistVideosTable.position to SortOrder.ASC, + ) + .associate { row -> + row[PlaylistVideosTable.playlistId] to row.toPlaylistVideoItem(emptyMap()) + } playlists.map { row -> - row.toPlaylistSummary(videoCounts[row[PlaylistsTable.id]] ?: 0) + val playlistId = row[PlaylistsTable.id] + row.toPlaylistSummary( + videoCount = videoCounts[playlistId] ?: 0, + firstVideo = firstVideos[playlistId], + ) } } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt index b6a40874..94a0baf8 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/PlaylistVideoProgressMapper.kt @@ -26,10 +26,14 @@ fun playlistProgressByUrl(userId: String, videoUrls: List): Map + repeat(12) { index -> val url = "https://www.youtube.com/watch?v=video$index" playlists.addVideo(TEST_USER_ID, playlist.id, fallbackVideo(url)) } @@ -158,11 +158,11 @@ class UserVideoMetadataRepairRoutesTest { try { repair.schedulePlaylists(scope, TEST_USER_ID) withTimeout(10_000) { - while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.count { it.title.startsWith("Resolved") } != 26) { + while (playlists.getById(TEST_USER_ID, playlist.id)?.videos?.count { it.title.startsWith("Resolved") } != 12) { delay(20) } } - assertEquals(26, attempts.get()) + assertEquals(12, attempts.get()) } finally { scope.cancel() } From bc07ddbb711c40a0dfe1c73ccab76b6f53382a69 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 21:43:28 +0200 Subject: [PATCH 131/137] feat: resolve missing subscription avatars --- .../server/routes/SubscriptionsRoutes.kt | 6 +- .../typetype/server/routes/UserDataRoutes.kt | 1 + .../dev/typetype/server/ServiceRegistry.kt | 2 + .../SubscriptionAvatarWarmupService.kt | 93 ++++++++++++++++++ .../SubscriptionAvatarWarmupServiceTest.kt | 97 +++++++++++++++++++ 5 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt create mode 100644 server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index b5355e9b..47627f1c 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionAvatarWarmupService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionSelection import dev.typetype.server.services.PushNotificationService @@ -28,6 +29,7 @@ internal fun Route.subscriptionsRoutes( warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), pushNotificationService: PushNotificationService? = null, + avatarWarmupService: SubscriptionAvatarWarmupService? = null, ) { subscriptionMembershipPageRoutes(authService, groupsService) get("/subscriptions/group-memberships") { @@ -48,7 +50,9 @@ internal fun Route.subscriptionsRoutes( ErrorResponse("Subscription group not found", "subscription_group_not_found"), ) } - call.respond(subscriptionsService.getAll(userId, selection)) + val subscriptions = subscriptionsService.getAll(userId, selection) + avatarWarmupService?.schedule(call.application, userId, subscriptions) + call.respond(subscriptions) } } post("/subscriptions") { diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 600edf47..5b40ebfd 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -26,6 +26,7 @@ internal fun Route.userDataRoutes( svc.homeRecommendationWarmupService, svc.subscriptionGroupsService, svc.pushNotificationService, + svc.subscriptionAvatarWarmupService, ) subscriptionFeedRoutes( svc.subscriptionFeedService, diff --git a/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index 25ee3519..4fd731f6 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -35,6 +35,7 @@ import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsBlendService import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionAvatarWarmupService import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidatorImpl @@ -105,6 +106,7 @@ class ServiceRegistry( val sabrSessionStore = extraction.sabrSessionStore val historyService = HistoryService() val subscriptionsService = SubscriptionsService() + val subscriptionAvatarWarmupService = SubscriptionAvatarWarmupService(channelService) val subscriptionGroupsService = SubscriptionGroupsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) val subscriptionShortsFeedService = SubscriptionShortsFeedService( diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt new file mode 100644 index 00000000..08fe1a78 --- /dev/null +++ b/server-services/src/main/kotlin/dev/typetype/server/services/SubscriptionAvatarWarmupService.kt @@ -0,0 +1,93 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubscriptionItem +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.update +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +class SubscriptionAvatarWarmupService(private val channelService: ChannelService) { + private val logger = LoggerFactory.getLogger(SubscriptionAvatarWarmupService::class.java) + private val inFlight = ConcurrentHashMap.newKeySet() + private val recentlyFailed = ConcurrentHashMap() + + fun schedule(scope: CoroutineScope, userId: String, items: List) { + val now = System.currentTimeMillis() + val candidates = items.asSequence() + .filter { it.avatarUrl.isBlank() } + .map(SubscriptionItem::channelUrl) + .distinct() + .filter { url -> + val failedAt = recentlyFailed[url] + failedAt == null || now - failedAt >= FAILURE_COOLDOWN_MS + } + .filter(inFlight::add) + .take(MAX_AVATARS_PER_REQUEST) + .toList() + if (candidates.isEmpty()) return + scope.launch { + try { + warm(userId, candidates) + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + logger.warn("Background subscription avatar warmup failed", error) + } finally { + inFlight.removeAll(candidates.toSet()) + } + } + } + + internal suspend fun warm(userId: String, channelUrls: List) { + channelUrls.chunked(MAX_CONCURRENT_LOOKUPS).forEach { chunk -> + coroutineScope { + chunk.map { channelUrl -> + async { resolve(userId, channelUrl) } + }.awaitAll() + } + delay(BATCH_DELAY_MS) + } + } + + private suspend fun resolve(userId: String, channelUrl: String) { + when (val result = channelService.getChannel(url = channelUrl, nextpage = null)) { + is ExtractionResult.Success -> { + val avatarUrl = result.data.avatarUrl.trim() + if (avatarUrl.isBlank()) { + recentlyFailed[channelUrl] = System.currentTimeMillis() + return + } + DatabaseFactory.query { + SubscriptionsTable.update({ + (SubscriptionsTable.userId eq userId) and + (SubscriptionsTable.channelUrl eq channelUrl) and + (SubscriptionsTable.avatarUrl eq "") + }) { + it[SubscriptionsTable.avatarUrl] = avatarUrl + } + } + recentlyFailed.remove(channelUrl) + } + is ExtractionResult.BadRequest, + is ExtractionResult.Failure -> recentlyFailed[channelUrl] = System.currentTimeMillis() + } + } + + private companion object { + const val MAX_AVATARS_PER_REQUEST = 8 + const val MAX_CONCURRENT_LOOKUPS = 2 + const val BATCH_DELAY_MS = 750L + const val FAILURE_COOLDOWN_MS = 15 * 60 * 1000L + } +} diff --git a/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt new file mode 100644 index 00000000..b8228a67 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/SubscriptionAvatarWarmupServiceTest.kt @@ -0,0 +1,97 @@ +package dev.typetype.server + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SubscriptionAvatarWarmupService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionAvatarWarmupServiceTest { + private val channelUrl = "https://www.youtube.com/channel/UCWarmup" + + companion object { + @BeforeAll + @JvmStatic + fun initDb(): Unit = TestDatabase.setup() + } + + @BeforeEach + fun clean(): Unit = TestDatabase.truncateAll() + + @Test + fun `warmup persists a resolved channel avatar`() = runBlocking { + insertSubscription() + val service = SubscriptionAvatarWarmupService(channelService("https://avatar.test/warmup.jpg")) + + service.warm(TEST_USER_ID, listOf(channelUrl)) + + assertEquals("https://avatar.test/warmup.jpg", storedAvatar()) + } + + @Test + fun `warmup leaves a blank avatar when channel resolution has no avatar`() = runBlocking { + insertSubscription() + val service = SubscriptionAvatarWarmupService(channelService("")) + + service.warm(TEST_USER_ID, listOf(channelUrl)) + + assertEquals("", storedAvatar()) + } + + private suspend fun insertSubscription(): Unit { + SubscriptionsService().add( + TEST_USER_ID, + SubscriptionItem( + channelUrl = channelUrl, + name = "OHIOBOSS SATOYU", + avatarUrl = "", + ), + ) + } + + private suspend fun storedAvatar(): String = DatabaseFactory.query { + SubscriptionsTable.selectAll() + .where { + (SubscriptionsTable.userId eq TEST_USER_ID) and + (SubscriptionsTable.channelUrl eq channelUrl) + } + .single()[SubscriptionsTable.avatarUrl] + } + + private fun channelService(avatarUrl: String): ChannelService = object : ChannelService { + override suspend fun getChannel( + url: String, + nextpage: String?, + sort: String?, + ): ExtractionResult = ExtractionResult.Success( + ChannelResponse( + name = "OHIOBOSS SATOYU", + description = "", + avatarUrl = avatarUrl, + bannerUrl = "", + subscriberCount = 0, + isVerified = false, + videos = emptyList(), + nextpage = null, + ), + ) + + override suspend fun getPlaylists( + url: String, + nextpage: String?, + ): ExtractionResult = + ExtractionResult.BadRequest("Not used") + } +} From 6ef40e58f2d06b70f2fe41af6cbaca44c4dfa415 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 22:24:00 +0200 Subject: [PATCH 132/137] feat: update provider support and playlist blocking --- build.gradle.kts | 2 +- server-admin/build.gradle.kts | 2 +- server-domain/build.gradle.kts | 2 +- server-downloader/build.gradle.kts | 2 +- server-http/build.gradle.kts | 2 +- .../dev/typetype/server/ApplicationRoutes.kt | 16 ++++- .../typetype/server/routes/ChannelRoutes.kt | 55 ++++++++++++--- .../server/routes/PublicPlaylistRoutes.kt | 15 +++- server-playback/build.gradle.kts | 2 +- server-sabr/build.gradle.kts | 2 +- server-services/build.gradle.kts | 2 +- .../server/services/BlockedContentFilters.kt | 19 +++++ .../typetype/server/ChannelTabResolverTest.kt | 8 +++ .../services/BlockedContentFiltersTest.kt | 70 +++++++++++++++++++ server-token-gateway/build.gradle.kts | 2 +- 15 files changed, 178 insertions(+), 23 deletions(-) create mode 100644 server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 629b8171..237542ec 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,7 +57,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts index a51af2ed..48c09776 100644 --- a/server-admin/build.gradle.kts +++ b/server-admin/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("org.slf4j:slf4j-api:2.0.16") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts index 47f68667..f748dbfe 100644 --- a/server-domain/build.gradle.kts +++ b/server-domain/build.gradle.kts @@ -7,7 +7,7 @@ dependencies { api(project(":server-core")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation(project(":server-cache")) implementation(project(":server-db")) implementation("org.jetbrains.exposed:exposed-core:1.5.0") diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts index 521c9a9c..fd40403a 100644 --- a/server-downloader/build.gradle.kts +++ b/server-downloader/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts index 023e827f..56a4a753 100644 --- a/server-http/build.gradle.kts +++ b/server-http/build.gradle.kts @@ -26,7 +26,7 @@ dependencies { implementation("io.ktor:ktor-utils-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 2e9422e3..29590bfa 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -86,12 +86,24 @@ fun Application.installApplicationRoutes( searchRoutes(svc.searchService, authService, svc.accessControlService, adminSettingsService, svc.blockedService) suggestionRoutes(svc.suggestionService, authService, adminSettingsService) trendingRoutes(svc.trendingService, authService, svc.accessControlService, adminSettingsService) - publicPlaylistRoutes(svc.publicPlaylistService, authService, svc.accessControlService, adminSettingsService) + publicPlaylistRoutes( + svc.publicPlaylistService, + authService, + svc.accessControlService, + adminSettingsService, + svc.blockedService, + ) commentRoutes(svc.commentService, authService, adminSettingsService) bulletCommentRoutes(svc.bulletCommentService, authService, adminSettingsService) } rateLimit(CHANNEL_ZONE) { - channelRoutes(svc.channelService, authService, svc.accessControlService, adminSettingsService) + channelRoutes( + svc.channelService, + authService, + svc.accessControlService, + adminSettingsService, + svc.blockedService, + ) podcastRoutes(svc.podcastService, authService, adminSettingsService) } installProxyRoutes(svc) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt index 75c25b87..ce89b38a 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/ChannelRoutes.kt @@ -9,8 +9,11 @@ import dev.typetype.server.services.AccessControlProfile import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedContentProfile +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.ChannelService import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -24,20 +27,32 @@ fun Route.channelRoutes( authService: AuthService? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, + blockedService: BlockedService? = null, ) { get("/channel") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] val sort = call.request.queryParameters["sort"]?.takeIf { it.isNotBlank() } - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty when (val result = channelService.getChannel(url = url, nextpage = nextpage, sort = sort)) { is ExtractionResult.Success -> { - if (!profile.allowsChannel(url = url, name = result.data.name)) { + if (!access.profile.allowsChannel(url = url, name = result.data.name)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - call.respond(result.data.filterAllowed(profile)) + if (!blocked.allowsChannel(url = url, name = result.data.name)) { + return@get call.respond( + HttpStatusCode.Forbidden, + ErrorResponse("Channel is blocked", "content_blocked"), + ) + } + call.respond(result.data.filterAllowed(access.profile).filterBlocked(blocked)) } is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) @@ -47,7 +62,12 @@ fun Route.channelRoutes( val request = call.receive() val url = request.url?.takeIf { it.isNotBlank() } ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@post + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@post + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty call.respondChannelResult( channelService.getChannel( @@ -56,19 +76,29 @@ fun Route.channelRoutes( sort = request.sort?.takeIf { it.isNotBlank() }, ), url, - profile, + access.profile, + blocked, ) } get("/channel/playlists") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get - if (!profile.allowsChannel(url)) { + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty + if (!access.profile.allowsChannel(url)) { return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - call.respondChannelPlaylistsResult(channelService.getPlaylists(url = url, nextpage = nextpage), profile) + call.respondChannelPlaylistsResult( + channelService.getPlaylists(url = url, nextpage = nextpage), + access.profile, + blocked, + ) } } @@ -76,13 +106,17 @@ private suspend fun ApplicationCall.respondChannelResult( result: ExtractionResult, url: String, profile: AccessControlProfile, + blocked: BlockedContentProfile, ) { when (result) { is ExtractionResult.Success -> { if (!profile.allowsChannel(url = url, name = result.data.name)) { return respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is not allowed")) } - respond(result.data.filterAllowed(profile)) + if (!blocked.allowsChannel(url = url, name = result.data.name)) { + return respond(HttpStatusCode.Forbidden, ErrorResponse("Channel is blocked", "content_blocked")) + } + respond(result.data.filterAllowed(profile).filterBlocked(blocked)) } is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) @@ -92,9 +126,10 @@ private suspend fun ApplicationCall.respondChannelResult( private suspend fun ApplicationCall.respondChannelPlaylistsResult( result: ExtractionResult, profile: AccessControlProfile, + blocked: BlockedContentProfile, ) { when (result) { - is ExtractionResult.Success -> respond(result.data.filterAllowed(profile)) + is ExtractionResult.Success -> respond(result.data.filterAllowed(profile).filterBlocked(blocked)) is ExtractionResult.BadRequest -> respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) } diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt index 79801e13..891ad657 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/PublicPlaylistRoutes.kt @@ -5,8 +5,11 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.AccessControlService import dev.typetype.server.services.AdminSettingsService import dev.typetype.server.services.AuthService +import dev.typetype.server.services.BlockedContentProfile +import dev.typetype.server.services.BlockedService import dev.typetype.server.services.PublicPlaylistService import dev.typetype.server.services.filterAllowed +import dev.typetype.server.services.filterBlocked import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond import io.ktor.server.routing.Route @@ -17,15 +20,23 @@ fun Route.publicPlaylistRoutes( authService: AuthService? = null, accessControlService: AccessControlService? = null, adminSettingsService: AdminSettingsService? = null, + blockedService: BlockedService? = null, ) { get("/playlist") { val url = call.request.queryParameters["url"] ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing 'url' parameter")) val nextpage = call.request.queryParameters["nextpage"] - val profile = call.accessProfileOrRespond(authService, accessControlService, adminSettingsService)?.profile ?: return@get + val access = call.accessProfileOrRespond( + authService, + accessControlService, + adminSettingsService, + ) ?: return@get + val blocked = access.userId?.let { blockedService?.profileFor(it) } ?: BlockedContentProfile.empty when (val result = playlistService.getPlaylist(url = url, nextpage = nextpage)) { - is ExtractionResult.Success -> call.respond(result.data.filterAllowed(profile)) + is ExtractionResult.Success -> call.respond( + result.data.filterAllowed(access.profile).filterBlocked(blocked), + ) is ExtractionResult.BadRequest -> call.respond(HttpStatusCode.BadRequest, ErrorResponse(result.message)) is ExtractionResult.Failure -> call.respond(HttpStatusCode.UnprocessableEntity, ErrorResponse(result.message)) } diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts index 4d05bd93..858be2f5 100644 --- a/server-playback/build.gradle.kts +++ b/server-playback/build.gradle.kts @@ -13,7 +13,7 @@ dependencies { implementation(project(":server-token-gateway")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts index 099629f1..ec2765e5 100644 --- a/server-sabr/build.gradle.kts +++ b/server-sabr/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - api("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + api("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts index ec0d6606..28f370ed 100644 --- a/server-services/build.gradle.kts +++ b/server-services/build.gradle.kts @@ -24,7 +24,7 @@ dependencies { implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") implementation("org.slf4j:slf4j-api:2.0.16") diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt index 76d52af9..8770d11e 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/BlockedContentFilters.kt @@ -1,6 +1,9 @@ package dev.typetype.server.services +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse import dev.typetype.server.models.HomeRecommendationsResponse +import dev.typetype.server.models.PublicPlaylistResponse import dev.typetype.server.models.SearchPageResponse import dev.typetype.server.models.StreamResponse @@ -21,3 +24,19 @@ fun StreamResponse.filterBlocked(profile: BlockedContentProfile): StreamResponse profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) }, ) + +fun PublicPlaylistResponse.filterBlocked(profile: BlockedContentProfile): PublicPlaylistResponse = copy( + videos = videos.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun ChannelResponse.filterBlocked(profile: BlockedContentProfile): ChannelResponse = copy( + videos = videos.filter { + profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) + }, +) + +fun ChannelPlaylistsResponse.filterBlocked(profile: BlockedContentProfile): ChannelPlaylistsResponse = copy( + playlists = playlists.filter { profile.allowsChannel(url = "", name = it.uploaderName) }, +) diff --git a/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt b/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt index 59cdf9e2..449954c3 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/ChannelTabResolverTest.kt @@ -16,6 +16,14 @@ class ChannelTabResolverTest { assertEquals("https://www.youtube.com/@test", url.toBaseChannelUrl(ChannelTabs.LIVESTREAMS)) } + @Test + fun `niconico livestreams url maps to livestreams tab`() { + val url = "https://www.nicovideo.jp/user/123/livestreams" + + assertEquals(ChannelTabs.LIVESTREAMS, url.toChannelTab(null)) + assertEquals("https://www.nicovideo.jp/user/123", url.toBaseChannelUrl(ChannelTabs.LIVESTREAMS)) + } + @Test fun `channel sort maps base url to videos tab`() { val url = "https://www.youtube.com/@test" diff --git a/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt new file mode 100644 index 00000000..d25627d2 --- /dev/null +++ b/server-services/src/test/kotlin/dev/typetype/server/services/BlockedContentFiltersTest.kt @@ -0,0 +1,70 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.BlockedItem +import dev.typetype.server.models.BlockedKeywordItem +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.PlaylistResultItem +import dev.typetype.server.models.PublicPlaylistItem +import dev.typetype.server.models.PublicPlaylistResponse +import dev.typetype.server.testVideoItem +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class BlockedContentFiltersTest { + private val blocked = BlockedContentProfile( + videos = listOf(BlockedItem("https://youtube.com/watch?v=blocked", blockedAt = 1)), + channels = listOf(BlockedItem("https://youtube.com/@blocked", "Blocked Channel", null, 1)), + keywords = listOf(BlockedKeywordItem("spoiler", 1)), + ) + + @Test + fun `public playlist hides blocked videos channels and keywords`() { + val response = PublicPlaylistResponse( + playlist = PublicPlaylistItem("playlist", "Playlist", "url", "", "Owner", 4, "normal"), + videos = listOf( + video("https://youtube.com/watch?v=blocked", "Visible"), + video("https://youtube.com/watch?v=channel", "Visible", "Blocked Channel"), + video("https://youtube.com/watch?v=keyword", "Major spoiler"), + video("https://youtube.com/watch?v=allowed", "Allowed"), + ), + nextpage = null, + ) + + assertEquals( + listOf("https://youtube.com/watch?v=allowed"), + response.filterBlocked(blocked).videos.map { it.url }, + ) + } + + @Test + fun `channel playlist list hides playlists from blocked uploaders`() { + val response = ChannelPlaylistsResponse( + playlists = listOf( + playlist("blocked", "Blocked Channel"), + playlist("allowed", "Allowed Channel"), + ), + nextpage = null, + ) + + assertEquals(listOf("allowed"), response.filterBlocked(blocked).playlists.map { it.id }) + } + + private fun video(url: String, title: String, uploaderName: String = "Allowed Channel") = + testVideoItem().copy( + id = url, + url = url, + title = title, + uploaderName = uploaderName, + uploaderUrl = "https://youtube.com/@${uploaderName.lowercase().replace(' ', '-')}", + ) + + private fun playlist(id: String, uploaderName: String) = PlaylistResultItem( + id = id, + title = id, + url = "https://youtube.com/playlist?list=$id", + thumbnailUrl = "", + uploaderName = uploaderName, + streamCount = 1, + playlistType = "normal", + ) +} diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts index 0dad5922..6649d742 100644 --- a/server-token-gateway/build.gradle.kts +++ b/server-token-gateway/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("org.slf4j:slf4j-api:2.0.16") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:1d8bf8a6a5dd47d9993b95895dd1be3bb397481a") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("io.ktor:ktor-client-core-jvm:3.5.2") From 6f80583712363323caa0a4ba8797d3bbee8b5078 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 26 Sep 2026 22:33:50 +0200 Subject: [PATCH 133/137] fix: allow BiliBili live HLS gateway --- .../main/kotlin/dev/typetype/server/services/UrlValidator.kt | 1 + .../src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt index 1d3efb67..95e7c1e3 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt @@ -75,6 +75,7 @@ fun providerForProxyHost(rawHost: String): ProxyProvider? { host.matchesHost("bilivideo.com") || host.matchesHost("bilivideo.cn") || host.matchesHost("hdslb.com") || + host.matchesHost("live.bilibili.com") || host == "upos-hz-mirrorakam.akamaized.net" -> ProxyProvider.BILIBILI host.matchesHost("nicovideo.jp") || host.matchesHost("nimg.jp") -> ProxyProvider.NICONICO else -> null diff --git a/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt b/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt index fc3d0252..19df7677 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt @@ -28,6 +28,10 @@ class UrlValidatorTest { assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://i.ytimg.com/image.jpg").provider) assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://yt3.googleusercontent.com/avatar").provider) assertEquals(ProxyProvider.BILIBILI, requireProxyTarget("https://i2.hdslb.com/image.jpg").provider) + assertEquals( + ProxyProvider.BILIBILI, + requireProxyTarget("https://api.live.bilibili.com/xlive/play-gateway/master/url?cid=1").provider, + ) assertEquals( ProxyProvider.BILIBILI, requireProxyTarget("https://upos-hz-mirrorakam.akamaized.net/video.m4s").provider, From 9d9b45ff9e76053c79da3fb8d085510a3d03c749 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 27 Sep 2026 08:11:07 +0200 Subject: [PATCH 134/137] fix: update PipePipeExtractor live handling --- build.gradle.kts | 2 +- server-admin/build.gradle.kts | 2 +- server-domain/build.gradle.kts | 2 +- server-downloader/build.gradle.kts | 2 +- server-http/build.gradle.kts | 2 +- server-playback/build.gradle.kts | 2 +- server-sabr/build.gradle.kts | 2 +- server-services/build.gradle.kts | 2 +- server-token-gateway/build.gradle.kts | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 237542ec..992461b4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,7 +57,7 @@ dependencies { implementation("io.ktor:ktor-server-call-logging-jvm") implementation("io.ktor:ktor-server-rate-limit-jvm") implementation("ch.qos.logback:logback-classic:1.6.3") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") implementation("org.json:json:20260814") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/server-admin/build.gradle.kts b/server-admin/build.gradle.kts index 48c09776..24207a8c 100644 --- a/server-admin/build.gradle.kts +++ b/server-admin/build.gradle.kts @@ -19,7 +19,7 @@ dependencies { implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("org.slf4j:slf4j-api:2.0.16") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-domain/build.gradle.kts b/server-domain/build.gradle.kts index f748dbfe..dc133b34 100644 --- a/server-domain/build.gradle.kts +++ b/server-domain/build.gradle.kts @@ -7,7 +7,7 @@ dependencies { api(project(":server-core")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation(project(":server-cache")) implementation(project(":server-db")) implementation("org.jetbrains.exposed:exposed-core:1.5.0") diff --git a/server-downloader/build.gradle.kts b/server-downloader/build.gradle.kts index fd40403a..2cedb894 100644 --- a/server-downloader/build.gradle.kts +++ b/server-downloader/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-http/build.gradle.kts b/server-http/build.gradle.kts index 56a4a753..bd57eee7 100644 --- a/server-http/build.gradle.kts +++ b/server-http/build.gradle.kts @@ -26,7 +26,7 @@ dependencies { implementation("io.ktor:ktor-utils-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("com.fasterxml.jackson.core:jackson-core:2.22.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") testImplementation("io.ktor:ktor-server-test-host-jvm:3.5.2") diff --git a/server-playback/build.gradle.kts b/server-playback/build.gradle.kts index 858be2f5..148855f3 100644 --- a/server-playback/build.gradle.kts +++ b/server-playback/build.gradle.kts @@ -13,7 +13,7 @@ dependencies { implementation(project(":server-token-gateway")) implementation(project(":server-downloader")) implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.squareup.okhttp3:okhttp:5.5.0") diff --git a/server-sabr/build.gradle.kts b/server-sabr/build.gradle.kts index ec2765e5..7266fc45 100644 --- a/server-sabr/build.gradle.kts +++ b/server-sabr/build.gradle.kts @@ -4,7 +4,7 @@ plugins { dependencies { compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - api("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + api("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testImplementation("io.mockk:mockk:1.14.11") testRuntimeOnly("org.junit.platform:junit-platform-launcher") diff --git a/server-services/build.gradle.kts b/server-services/build.gradle.kts index 28f370ed..1270b6fb 100644 --- a/server-services/build.gradle.kts +++ b/server-services/build.gradle.kts @@ -24,7 +24,7 @@ dependencies { implementation("io.ktor:ktor-client-okhttp-jvm:3.5.2") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("com.squareup.okhttp3:okhttp:5.5.0") implementation("org.json:json:20260814") implementation("org.slf4j:slf4j-api:2.0.16") diff --git a/server-token-gateway/build.gradle.kts b/server-token-gateway/build.gradle.kts index 6649d742..195077a4 100644 --- a/server-token-gateway/build.gradle.kts +++ b/server-token-gateway/build.gradle.kts @@ -10,7 +10,7 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.2") implementation("org.slf4j:slf4j-api:2.0.16") compileOnly("com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751") - implementation("com.github.Priveetee.PipePipeExtractor:extractor:4a4e48ff762ea60cf2915adf57ec6d94bd86f2a4") + implementation("com.github.Priveetee.PipePipeExtractor:extractor:4eebe6d405122f0f63b7690c41f88f6d343a66fb") implementation("io.ktor:ktor-server-core-jvm:3.5.2") implementation("io.ktor:ktor-server-websockets-jvm:3.5.2") implementation("io.ktor:ktor-client-core-jvm:3.5.2") From 4381324128ba27b5efdf086fecbedcdbb09cff4f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 27 Sep 2026 08:30:30 +0200 Subject: [PATCH 135/137] fix: stabilize DeArrow metadata and thumbnails --- .../typetype/server/routes/DeArrowRoutes.kt | 13 ++++- .../typetype/server/services/DeArrowClient.kt | 44 ++++++++++++--- .../server/services/DeArrowService.kt | 17 ++++-- .../dev/typetype/server/DeArrowServiceTest.kt | 53 +++++++++++++++++-- 4 files changed, 109 insertions(+), 18 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt index a85979d3..f557438f 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt @@ -1,6 +1,7 @@ package dev.typetype.server.routes import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.services.DeArrowUnavailableException import dev.typetype.server.services.DeArrowService import io.ktor.http.ContentType import io.ktor.http.HttpHeaders @@ -13,8 +14,16 @@ import io.ktor.server.routing.get fun Route.deArrowRoutes(service: DeArrowService) { get("/dearrow") { val videoId = call.request.queryParameters["videoId"].orEmpty() - val item = service.get(videoId) - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoId")) + val item = try { + service.get(videoId) + ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoId")) + } catch (_: DeArrowUnavailableException) { + call.response.headers.append(HttpHeaders.RetryAfter, "30") + return@get call.respond( + HttpStatusCode.ServiceUnavailable, + ErrorResponse("DeArrow is temporarily unavailable", "dearrow_unavailable"), + ) + } call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=3600") call.respond(item) } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt index edd5e0a7..c270d75e 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowClient.kt @@ -1,28 +1,58 @@ package dev.typetype.server.services import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import okhttp3.OkHttpClient import okhttp3.Request +import java.util.concurrent.TimeUnit interface DeArrowRemote { suspend fun branding(videoId: String): String? suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? + suspend fun fallbackThumbnail(videoId: String): ByteArray? } -class DeArrowClient(private val client: OkHttpClient = OkHttpClient()) : DeArrowRemote { +class DeArrowClient( + private val client: OkHttpClient = defaultClient(), +) : DeArrowRemote { + private val limiter = Semaphore(MAX_CONCURRENT_REQUESTS) + override suspend fun branding(videoId: String): String? = get("https://sponsor.ajay.app/api/branding?videoID=$videoId&fetchAll=true")?.decodeToString() override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? = get("https://dearrow-thumb.ajay.app/api/v1/getThumbnail?videoID=$videoId&time=$timestamp") + override suspend fun fallbackThumbnail(videoId: String): ByteArray? = + get("https://i.ytimg.com/vi/$videoId/hqdefault.jpg") + private suspend fun get(url: String): ByteArray? = withContext(Dispatchers.IO) { - runCatching { - client.newCall(Request.Builder().url(url).get().build()).execute().use { response -> - if (!response.isSuccessful) return@use null - response.body.bytes().takeIf { it.isNotEmpty() } - } - }.getOrNull() + limiter.withPermit { + runCatching { + client.newCall( + Request.Builder() + .url(url) + .header("User-Agent", USER_AGENT) + .get() + .build(), + ).execute().use { response -> + if (!response.isSuccessful) return@use null + response.body.bytes().takeIf { it.isNotEmpty() } + } + }.getOrNull() + } + } + + private companion object { + private const val MAX_CONCURRENT_REQUESTS = 6 + private const val USER_AGENT = "TypeType-DeArrow/1.0" + + fun defaultClient(): OkHttpClient = OkHttpClient.Builder() + .connectTimeout(4, TimeUnit.SECONDS) + .readTimeout(8, TimeUnit.SECONDS) + .callTimeout(10, TimeUnit.SECONDS) + .build() } } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt index f354b6d2..8d3b0bfb 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt @@ -14,18 +14,21 @@ import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import java.util.Base64 +class DeArrowUnavailableException : RuntimeException("DeArrow is temporarily unavailable") + class DeArrowService( private val cache: CacheService, private val client: DeArrowRemote = DeArrowClient(), ) { suspend fun get(videoId: String): DeArrowItem? { if (!isValidVideoId(videoId)) return null - cache.get("dearrow:branding:v2:$videoId")?.let { + cache.get("dearrow:branding:v3:$videoId")?.let { return runCatching { CacheJson.decodeFromString(DeArrowItem.serializer(), it) }.getOrNull() } - val item = client.branding(videoId)?.let { parse(videoId, it) } ?: DeArrowItem(videoId) + val raw = client.branding(videoId) ?: throw DeArrowUnavailableException() + val item = parse(videoId, raw) cache.set( - "dearrow:branding:v2:$videoId", + "dearrow:branding:v3:$videoId", CacheJson.encodeToString(DeArrowItem.serializer(), item), BRANDING_TTL_SECONDS, ) @@ -37,8 +40,11 @@ class DeArrowService( val normalizedTime = "%.3f".format(java.util.Locale.ROOT, timestamp) val key = "dearrow:thumbnail:$videoId:$normalizedTime" cache.get(key)?.let { return runCatching { Base64.getDecoder().decode(it) }.getOrNull() } - val bytes = client.thumbnail(videoId, timestamp)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } ?: return null - cache.set(key, Base64.getEncoder().encodeToString(bytes), THUMBNAIL_TTL_SECONDS) + val generated = client.thumbnail(videoId, timestamp)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } + val fallback = generated ?: client.fallbackThumbnail(videoId)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } + val bytes = fallback ?: return null + val ttl = if (generated != null) THUMBNAIL_TTL_SECONDS else FALLBACK_THUMBNAIL_TTL_SECONDS + cache.set(key, Base64.getEncoder().encodeToString(bytes), ttl) return bytes } @@ -115,6 +121,7 @@ class DeArrowService( companion object { private const val BRANDING_TTL_SECONDS = 86_400L private const val THUMBNAIL_TTL_SECONDS = 604_800L + private const val FALLBACK_THUMBNAIL_TTL_SECONDS = 900L private const val MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024 private val VIDEO_ID_REGEX = Regex("^[A-Za-z0-9_-]{11}$") private val TITLE_MARKER_REGEX = Regex(">(?=\\S)") diff --git a/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt index bd1bcff4..01e5b164 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt @@ -1,10 +1,14 @@ package dev.typetype.server +import dev.typetype.server.services.DeArrowUnavailableException import dev.typetype.server.services.DeArrowRemote import dev.typetype.server.services.DeArrowService import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class DeArrowServiceTest { @@ -32,19 +36,60 @@ class DeArrowServiceTest { assertNull(service.get("invalid")) assertEquals(0, remote.brandingCalls) } + + @Test + fun `does not cache upstream failures`() = runBlocking { + val cache = FakeCacheService() + val remote = FakeDeArrowRemote().apply { brandingResult = null } + val service = DeArrowService(cache, remote) + + assertThrows(DeArrowUnavailableException::class.java) { + runBlocking { service.get("stZ3ZoR_8eg") } + } + assertTrue(cache.keys().isEmpty()) + + remote.brandingResult = FakeDeArrowRemote.BRANDING + assertEquals("Clear title", service.get("stZ3ZoR_8eg")?.title) + assertEquals(2, remote.brandingCalls) + } + + @Test + fun `falls back to the original thumbnail when generation fails`() = runBlocking { + val cache = FakeCacheService() + val remote = FakeDeArrowRemote().apply { thumbnailResult = null } + val service = DeArrowService(cache, remote) + + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, service.thumbnail("stZ3ZoR_8eg", 12.5)) + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, service.thumbnail("stZ3ZoR_8eg", 12.5)) + assertEquals(1, remote.thumbnailCalls) + assertEquals(1, remote.fallbackThumbnailCalls) + } } private class FakeDeArrowRemote : DeArrowRemote { var brandingCalls = 0 + var thumbnailCalls = 0 + var fallbackThumbnailCalls = 0 + var brandingResult: String? = BRANDING + var thumbnailResult: ByteArray? = byteArrayOf(1, 2, 3) - override suspend fun branding(videoId: String): String { + override suspend fun branding(videoId: String): String? { brandingCalls += 1 - return BRANDING + return brandingResult } - override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray = byteArrayOf(1, 2, 3) + override suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? { + thumbnailCalls += 1 + return thumbnailResult + } + + override suspend fun fallbackThumbnail(videoId: String): ByteArray { + fallbackThumbnailCalls += 1 + return FALLBACK_THUMBNAIL + } companion object { - private const val BRANDING = """{"titles":[{"title":"Rejected title","votes":-2,"locked":false,"original":false,"UUID":"rejected"},{"title":"Clear title","votes":2,"locked":false,"original":false,"UUID":"accepted"},{"title":"Original title","votes":0,"locked":false,"original":true,"UUID":"original"}],"thumbnails":[{"timestamp":8.0,"votes":-1,"locked":false,"original":false,"UUID":"rejected-thumb"},{"timestamp":12.5,"votes":1,"locked":false,"original":false,"UUID":"accepted-thumb"},{"votes":0,"locked":false,"original":true,"UUID":"original-thumb"}],"videoDuration":100,"randomTime":0.4}""" + const val BRANDING = """{"titles":[{"title":"Rejected title","votes":-2,"locked":false,"original":false,"UUID":"rejected"},{"title":"Clear title","votes":2,"locked":false,"original":false,"UUID":"accepted"},{"title":"Original title","votes":0,"locked":false,"original":true,"UUID":"original"}],"thumbnails":[{"timestamp":8.0,"votes":-1,"locked":false,"original":false,"UUID":"rejected-thumb"},{"timestamp":12.5,"votes":1,"locked":false,"original":false,"UUID":"accepted-thumb"},{"votes":0,"locked":false,"original":true,"UUID":"original-thumb"}],"videoDuration":100,"randomTime":0.4}""" + val FALLBACK_THUMBNAIL = byteArrayOf(9, 8, 7) } } From b38db9f4ac2e4efbfaedd84c4aaf0a2174210235 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 27 Sep 2026 08:40:02 +0200 Subject: [PATCH 136/137] fix: expire DeArrow thumbnail fallback quickly --- .../typetype/server/routes/DeArrowRoutes.kt | 7 ++--- .../server/services/DeArrowService.kt | 27 +++++++++++++++---- .../dev/typetype/server/DeArrowServiceTest.kt | 8 ++++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt index f557438f..fed86656 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/routes/DeArrowRoutes.kt @@ -31,10 +31,11 @@ fun Route.deArrowRoutes(service: DeArrowService) { val videoId = call.request.queryParameters["videoId"].orEmpty() val timestamp = call.request.queryParameters["time"]?.toDoubleOrNull() ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid time")) - val bytes = service.thumbnail(videoId, timestamp) + val thumbnail = service.thumbnail(videoId, timestamp) ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Thumbnail not found")) - call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=604800") - call.respondBytes(bytes, deArrowThumbnailContentType(bytes)) + val maxAge = if (thumbnail.fallback) 900 else 604800 + call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=$maxAge") + call.respondBytes(thumbnail.bytes, deArrowThumbnailContentType(thumbnail.bytes)) } } diff --git a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt index 8d3b0bfb..a1d0805d 100644 --- a/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt +++ b/server-services/src/main/kotlin/dev/typetype/server/services/DeArrowService.kt @@ -16,6 +16,11 @@ import java.util.Base64 class DeArrowUnavailableException : RuntimeException("DeArrow is temporarily unavailable") +data class DeArrowThumbnail( + val bytes: ByteArray, + val fallback: Boolean, +) + class DeArrowService( private val cache: CacheService, private val client: DeArrowRemote = DeArrowClient(), @@ -35,17 +40,27 @@ class DeArrowService( return item } - suspend fun thumbnail(videoId: String, timestamp: Double): ByteArray? { + suspend fun thumbnail(videoId: String, timestamp: Double): DeArrowThumbnail? { if (!isValidVideoId(videoId) || !timestamp.isFinite() || timestamp < 0.0) return null val normalizedTime = "%.3f".format(java.util.Locale.ROOT, timestamp) val key = "dearrow:thumbnail:$videoId:$normalizedTime" - cache.get(key)?.let { return runCatching { Base64.getDecoder().decode(it) }.getOrNull() } + cache.get(key)?.let { return decodeThumbnail(it) } val generated = client.thumbnail(videoId, timestamp)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } val fallback = generated ?: client.fallbackThumbnail(videoId)?.takeIf { it.size <= MAX_THUMBNAIL_BYTES } val bytes = fallback ?: return null - val ttl = if (generated != null) THUMBNAIL_TTL_SECONDS else FALLBACK_THUMBNAIL_TTL_SECONDS - cache.set(key, Base64.getEncoder().encodeToString(bytes), ttl) - return bytes + val isFallback = generated == null + val ttl = if (isFallback) FALLBACK_THUMBNAIL_TTL_SECONDS else THUMBNAIL_TTL_SECONDS + val prefix = if (isFallback) FALLBACK_CACHE_PREFIX else GENERATED_CACHE_PREFIX + cache.set(key, prefix + Base64.getEncoder().encodeToString(bytes), ttl) + return DeArrowThumbnail(bytes, isFallback) + } + + private fun decodeThumbnail(value: String): DeArrowThumbnail? { + val fallback = value.startsWith(FALLBACK_CACHE_PREFIX) + val payload = value.removePrefix(FALLBACK_CACHE_PREFIX).removePrefix(GENERATED_CACHE_PREFIX) + return runCatching { + DeArrowThumbnail(Base64.getDecoder().decode(payload), fallback) + }.getOrNull() } private fun parse(videoId: String, raw: String): DeArrowItem { @@ -123,6 +138,8 @@ class DeArrowService( private const val THUMBNAIL_TTL_SECONDS = 604_800L private const val FALLBACK_THUMBNAIL_TTL_SECONDS = 900L private const val MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024 + private const val FALLBACK_CACHE_PREFIX = "fallback:" + private const val GENERATED_CACHE_PREFIX = "generated:" private val VIDEO_ID_REGEX = Regex("^[A-Za-z0-9_-]{11}$") private val TITLE_MARKER_REGEX = Regex(">(?=\\S)") } diff --git a/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt index 01e5b164..ece0601a 100644 --- a/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt +++ b/server-services/src/test/kotlin/dev/typetype/server/DeArrowServiceTest.kt @@ -59,8 +59,12 @@ class DeArrowServiceTest { val remote = FakeDeArrowRemote().apply { thumbnailResult = null } val service = DeArrowService(cache, remote) - assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, service.thumbnail("stZ3ZoR_8eg", 12.5)) - assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, service.thumbnail("stZ3ZoR_8eg", 12.5)) + val first = service.thumbnail("stZ3ZoR_8eg", 12.5) + val second = service.thumbnail("stZ3ZoR_8eg", 12.5) + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, first?.bytes) + assertEquals(true, first?.fallback) + assertArrayEquals(FakeDeArrowRemote.FALLBACK_THUMBNAIL, second?.bytes) + assertEquals(true, second?.fallback) assertEquals(1, remote.thumbnailCalls) assertEquals(1, remote.fallbackThumbnailCalls) } From 7cc91d65e48485f4b139ebadf7564efb899c3bd1 Mon Sep 17 00:00:00 2001 From: Priveetee <138765817+Priveetee@users.noreply.github.com> Date: Sun, 27 Sep 2026 14:12:40 +0000 Subject: [PATCH 137/137] style: trim trailing blank lines --- .../src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt | 1 - src/main/kotlin/dev/typetype/server/Plugins.kt | 3 --- 2 files changed, 4 deletions(-) diff --git a/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt index 5230ec02..9e102365 100644 --- a/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt +++ b/server-http/src/main/kotlin/dev/typetype/server/StatusPagesConfig.kt @@ -40,4 +40,3 @@ fun Application.configureStatusPages() { } } } - diff --git a/src/main/kotlin/dev/typetype/server/Plugins.kt b/src/main/kotlin/dev/typetype/server/Plugins.kt index a092e265..c9c6a102 100644 --- a/src/main/kotlin/dev/typetype/server/Plugins.kt +++ b/src/main/kotlin/dev/typetype/server/Plugins.kt @@ -94,6 +94,3 @@ fun Application.configurePlugins(authService: AuthService) { } configureStatusPages() } - - -